PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
jsonb_util.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * jsonb_util.c
4  * converting between Jsonb and JsonbValues, and iterating.
5  *
6  * Copyright (c) 2014-2017, PostgreSQL Global Development Group
7  *
8  *
9  * IDENTIFICATION
10  * src/backend/utils/adt/jsonb_util.c
11  *
12  *-------------------------------------------------------------------------
13  */
14 #include "postgres.h"
15 
16 #include "access/hash.h"
17 #include "catalog/pg_collation.h"
18 #include "miscadmin.h"
19 #include "utils/builtins.h"
20 #include "utils/jsonb.h"
21 #include "utils/memutils.h"
22 #include "utils/varlena.h"
23 
24 /*
25  * Maximum number of elements in an array (or key/value pairs in an object).
26  * This is limited by two things: the size of the JEntry array must fit
27  * in MaxAllocSize, and the number of elements (or pairs) must fit in the bits
28  * reserved for that in the JsonbContainer.header field.
29  *
30  * (The total size of an array's or object's elements is also limited by
31  * JENTRY_OFFLENMASK, but we're not concerned about that here.)
32  */
33 #define JSONB_MAX_ELEMS (Min(MaxAllocSize / sizeof(JsonbValue), JB_CMASK))
34 #define JSONB_MAX_PAIRS (Min(MaxAllocSize / sizeof(JsonbPair), JB_CMASK))
35 
36 static void fillJsonbValue(JsonbContainer *container, int index,
37  char *base_addr, uint32 offset,
39 static bool equalsJsonbScalarValue(JsonbValue *a, JsonbValue *b);
42 static void convertJsonbValue(StringInfo buffer, JEntry *header, JsonbValue *val, int level);
43 static void convertJsonbArray(StringInfo buffer, JEntry *header, JsonbValue *val, int level);
44 static void convertJsonbObject(StringInfo buffer, JEntry *header, JsonbValue *val, int level);
45 static void convertJsonbScalar(StringInfo buffer, JEntry *header, JsonbValue *scalarVal);
46 
47 static int reserveFromBuffer(StringInfo buffer, int len);
48 static void appendToBuffer(StringInfo buffer, const char *data, int len);
49 static void copyToBuffer(StringInfo buffer, int offset, const char *data, int len);
50 static short padBufferToInt(StringInfo buffer);
51 
54 static JsonbParseState *pushState(JsonbParseState **pstate);
55 static void appendKey(JsonbParseState *pstate, JsonbValue *scalarVal);
56 static void appendValue(JsonbParseState *pstate, JsonbValue *scalarVal);
57 static void appendElement(JsonbParseState *pstate, JsonbValue *scalarVal);
58 static int lengthCompareJsonbStringValue(const void *a, const void *b);
59 static int lengthCompareJsonbPair(const void *a, const void *b, void *arg);
60 static void uniqueifyJsonbObject(JsonbValue *object);
63  JsonbValue *scalarVal);
64 
65 /*
66  * Turn an in-memory JsonbValue into a Jsonb for on-disk storage.
67  *
68  * There isn't a JsonbToJsonbValue(), because generally we find it more
69  * convenient to directly iterate through the Jsonb representation and only
70  * really convert nested scalar values. JsonbIteratorNext() does this, so that
71  * clients of the iteration code don't have to directly deal with the binary
72  * representation (JsonbDeepContains() is a notable exception, although all
73  * exceptions are internal to this module). In general, functions that accept
74  * a JsonbValue argument are concerned with the manipulation of scalar values,
75  * or simple containers of scalar values, where it would be inconvenient to
76  * deal with a great amount of other state.
77  */
78 Jsonb *
80 {
81  Jsonb *out;
82 
83  if (IsAJsonbScalar(val))
84  {
85  /* Scalar value */
86  JsonbParseState *pstate = NULL;
87  JsonbValue *res;
88  JsonbValue scalarArray;
89 
90  scalarArray.type = jbvArray;
91  scalarArray.val.array.rawScalar = true;
92  scalarArray.val.array.nElems = 1;
93 
94  pushJsonbValue(&pstate, WJB_BEGIN_ARRAY, &scalarArray);
95  pushJsonbValue(&pstate, WJB_ELEM, val);
96  res = pushJsonbValue(&pstate, WJB_END_ARRAY, NULL);
97 
98  out = convertToJsonb(res);
99  }
100  else if (val->type == jbvObject || val->type == jbvArray)
101  {
102  out = convertToJsonb(val);
103  }
104  else
105  {
106  Assert(val->type == jbvBinary);
107  out = palloc(VARHDRSZ + val->val.binary.len);
108  SET_VARSIZE(out, VARHDRSZ + val->val.binary.len);
109  memcpy(VARDATA(out), val->val.binary.data, val->val.binary.len);
110  }
111 
112  return out;
113 }
114 
115 /*
116  * Get the offset of the variable-length portion of a Jsonb node within
117  * the variable-length-data part of its container. The node is identified
118  * by index within the container's JEntry array.
119  */
120 uint32
122 {
123  uint32 offset = 0;
124  int i;
125 
126  /*
127  * Start offset of this entry is equal to the end offset of the previous
128  * entry. Walk backwards to the most recent entry stored as an end
129  * offset, returning that offset plus any lengths in between.
130  */
131  for (i = index - 1; i >= 0; i--)
132  {
133  offset += JBE_OFFLENFLD(jc->children[i]);
134  if (JBE_HAS_OFF(jc->children[i]))
135  break;
136  }
137 
138  return offset;
139 }
140 
141 /*
142  * Get the length of the variable-length portion of a Jsonb node.
143  * The node is identified by index within the container's JEntry array.
144  */
145 uint32
147 {
148  uint32 off;
149  uint32 len;
150 
151  /*
152  * If the length is stored directly in the JEntry, just return it.
153  * Otherwise, get the begin offset of the entry, and subtract that from
154  * the stored end+1 offset.
155  */
156  if (JBE_HAS_OFF(jc->children[index]))
157  {
158  off = getJsonbOffset(jc, index);
159  len = JBE_OFFLENFLD(jc->children[index]) - off;
160  }
161  else
162  len = JBE_OFFLENFLD(jc->children[index]);
163 
164  return len;
165 }
166 
167 /*
168  * BT comparator worker function. Returns an integer less than, equal to, or
169  * greater than zero, indicating whether a is less than, equal to, or greater
170  * than b. Consistent with the requirements for a B-Tree operator class
171  *
172  * Strings are compared lexically, in contrast with other places where we use a
173  * much simpler comparator logic for searching through Strings. Since this is
174  * called from B-Tree support function 1, we're careful about not leaking
175  * memory here.
176  */
177 int
179 {
180  JsonbIterator *ita,
181  *itb;
182  int res = 0;
183 
184  ita = JsonbIteratorInit(a);
185  itb = JsonbIteratorInit(b);
186 
187  do
188  {
189  JsonbValue va,
190  vb;
192  rb;
193 
194  ra = JsonbIteratorNext(&ita, &va, false);
195  rb = JsonbIteratorNext(&itb, &vb, false);
196 
197  if (ra == rb)
198  {
199  if (ra == WJB_DONE)
200  {
201  /* Decisively equal */
202  break;
203  }
204 
205  if (ra == WJB_END_ARRAY || ra == WJB_END_OBJECT)
206  {
207  /*
208  * There is no array or object to compare at this stage of
209  * processing. jbvArray/jbvObject values are compared
210  * initially, at the WJB_BEGIN_ARRAY and WJB_BEGIN_OBJECT
211  * tokens.
212  */
213  continue;
214  }
215 
216  if (va.type == vb.type)
217  {
218  switch (va.type)
219  {
220  case jbvString:
221  case jbvNull:
222  case jbvNumeric:
223  case jbvBool:
224  res = compareJsonbScalarValue(&va, &vb);
225  break;
226  case jbvArray:
227 
228  /*
229  * This could be a "raw scalar" pseudo array. That's
230  * a special case here though, since we still want the
231  * general type-based comparisons to apply, and as far
232  * as we're concerned a pseudo array is just a scalar.
233  */
234  if (va.val.array.rawScalar != vb.val.array.rawScalar)
235  res = (va.val.array.rawScalar) ? -1 : 1;
236  if (va.val.array.nElems != vb.val.array.nElems)
237  res = (va.val.array.nElems > vb.val.array.nElems) ? 1 : -1;
238  break;
239  case jbvObject:
240  if (va.val.object.nPairs != vb.val.object.nPairs)
241  res = (va.val.object.nPairs > vb.val.object.nPairs) ? 1 : -1;
242  break;
243  case jbvBinary:
244  elog(ERROR, "unexpected jbvBinary value");
245  }
246  }
247  else
248  {
249  /* Type-defined order */
250  res = (va.type > vb.type) ? 1 : -1;
251  }
252  }
253  else
254  {
255  /*
256  * It's safe to assume that the types differed, and that the va
257  * and vb values passed were set.
258  *
259  * If the two values were of the same container type, then there'd
260  * have been a chance to observe the variation in the number of
261  * elements/pairs (when processing WJB_BEGIN_OBJECT, say). They're
262  * either two heterogeneously-typed containers, or a container and
263  * some scalar type.
264  *
265  * We don't have to consider the WJB_END_ARRAY and WJB_END_OBJECT
266  * cases here, because we would have seen the corresponding
267  * WJB_BEGIN_ARRAY and WJB_BEGIN_OBJECT tokens first, and
268  * concluded that they don't match.
269  */
270  Assert(ra != WJB_END_ARRAY && ra != WJB_END_OBJECT);
271  Assert(rb != WJB_END_ARRAY && rb != WJB_END_OBJECT);
272 
273  Assert(va.type != vb.type);
274  Assert(va.type != jbvBinary);
275  Assert(vb.type != jbvBinary);
276  /* Type-defined order */
277  res = (va.type > vb.type) ? 1 : -1;
278  }
279  }
280  while (res == 0);
281 
282  while (ita != NULL)
283  {
284  JsonbIterator *i = ita->parent;
285 
286  pfree(ita);
287  ita = i;
288  }
289  while (itb != NULL)
290  {
291  JsonbIterator *i = itb->parent;
292 
293  pfree(itb);
294  itb = i;
295  }
296 
297  return res;
298 }
299 
300 /*
301  * Find value in object (i.e. the "value" part of some key/value pair in an
302  * object), or find a matching element if we're looking through an array. Do
303  * so on the basis of equality of the object keys only, or alternatively
304  * element values only, with a caller-supplied value "key". The "flags"
305  * argument allows the caller to specify which container types are of interest.
306  *
307  * This exported utility function exists to facilitate various cases concerned
308  * with "containment". If asked to look through an object, the caller had
309  * better pass a Jsonb String, because their keys can only be strings.
310  * Otherwise, for an array, any type of JsonbValue will do.
311  *
312  * In order to proceed with the search, it is necessary for callers to have
313  * both specified an interest in exactly one particular container type with an
314  * appropriate flag, as well as having the pointed-to Jsonb container be of
315  * one of those same container types at the top level. (Actually, we just do
316  * whichever makes sense to save callers the trouble of figuring it out - at
317  * most one can make sense, because the container either points to an array
318  * (possibly a "raw scalar" pseudo array) or an object.)
319  *
320  * Note that we can return a jbvBinary JsonbValue if this is called on an
321  * object, but we never do so on an array. If the caller asks to look through
322  * a container type that is not of the type pointed to by the container,
323  * immediately fall through and return NULL. If we cannot find the value,
324  * return NULL. Otherwise, return palloc()'d copy of value.
325  */
326 JsonbValue *
328  JsonbValue *key)
329 {
330  JEntry *children = container->children;
331  int count = JsonContainerSize(container);
333 
334  Assert((flags & ~(JB_FARRAY | JB_FOBJECT)) == 0);
335 
336  /* Quick out without a palloc cycle if object/array is empty */
337  if (count <= 0)
338  return NULL;
339 
340  result = palloc(sizeof(JsonbValue));
341 
342  if ((flags & JB_FARRAY) && JsonContainerIsArray(container))
343  {
344  char *base_addr = (char *) (children + count);
345  uint32 offset = 0;
346  int i;
347 
348  for (i = 0; i < count; i++)
349  {
350  fillJsonbValue(container, i, base_addr, offset, result);
351 
352  if (key->type == result->type)
353  {
354  if (equalsJsonbScalarValue(key, result))
355  return result;
356  }
357 
358  JBE_ADVANCE_OFFSET(offset, children[i]);
359  }
360  }
361  else if ((flags & JB_FOBJECT) && JsonContainerIsObject(container))
362  {
363  /* Since this is an object, account for *Pairs* of Jentrys */
364  char *base_addr = (char *) (children + count * 2);
365  uint32 stopLow = 0,
366  stopHigh = count;
367 
368  /* Object key passed by caller must be a string */
369  Assert(key->type == jbvString);
370 
371  /* Binary search on object/pair keys *only* */
372  while (stopLow < stopHigh)
373  {
374  uint32 stopMiddle;
375  int difference;
376  JsonbValue candidate;
377 
378  stopMiddle = stopLow + (stopHigh - stopLow) / 2;
379 
380  candidate.type = jbvString;
381  candidate.val.string.val =
382  base_addr + getJsonbOffset(container, stopMiddle);
383  candidate.val.string.len = getJsonbLength(container, stopMiddle);
384 
385  difference = lengthCompareJsonbStringValue(&candidate, key);
386 
387  if (difference == 0)
388  {
389  /* Found our key, return corresponding value */
390  int index = stopMiddle + count;
391 
392  fillJsonbValue(container, index, base_addr,
393  getJsonbOffset(container, index),
394  result);
395 
396  return result;
397  }
398  else
399  {
400  if (difference < 0)
401  stopLow = stopMiddle + 1;
402  else
403  stopHigh = stopMiddle;
404  }
405  }
406  }
407 
408  /* Not found */
409  pfree(result);
410  return NULL;
411 }
412 
413 /*
414  * Get i-th value of a Jsonb array.
415  *
416  * Returns palloc()'d copy of the value, or NULL if it does not exist.
417  */
418 JsonbValue *
420 {
422  char *base_addr;
423  uint32 nelements;
424 
425  if (!JsonContainerIsArray(container))
426  elog(ERROR, "not a jsonb array");
427 
428  nelements = JsonContainerSize(container);
429  base_addr = (char *) &container->children[nelements];
430 
431  if (i >= nelements)
432  return NULL;
433 
434  result = palloc(sizeof(JsonbValue));
435 
436  fillJsonbValue(container, i, base_addr,
437  getJsonbOffset(container, i),
438  result);
439 
440  return result;
441 }
442 
443 /*
444  * A helper function to fill in a JsonbValue to represent an element of an
445  * array, or a key or value of an object.
446  *
447  * The node's JEntry is at container->children[index], and its variable-length
448  * data is at base_addr + offset. We make the caller determine the offset
449  * since in many cases the caller can amortize that work across multiple
450  * children. When it can't, it can just call getJsonbOffset().
451  *
452  * A nested array or object will be returned as jbvBinary, ie. it won't be
453  * expanded.
454  */
455 static void
457  char *base_addr, uint32 offset,
459 {
460  JEntry entry = container->children[index];
461 
462  if (JBE_ISNULL(entry))
463  {
464  result->type = jbvNull;
465  }
466  else if (JBE_ISSTRING(entry))
467  {
468  result->type = jbvString;
469  result->val.string.val = base_addr + offset;
470  result->val.string.len = getJsonbLength(container, index);
471  Assert(result->val.string.len >= 0);
472  }
473  else if (JBE_ISNUMERIC(entry))
474  {
475  result->type = jbvNumeric;
476  result->val.numeric = (Numeric) (base_addr + INTALIGN(offset));
477  }
478  else if (JBE_ISBOOL_TRUE(entry))
479  {
480  result->type = jbvBool;
481  result->val.boolean = true;
482  }
483  else if (JBE_ISBOOL_FALSE(entry))
484  {
485  result->type = jbvBool;
486  result->val.boolean = false;
487  }
488  else
489  {
490  Assert(JBE_ISCONTAINER(entry));
491  result->type = jbvBinary;
492  /* Remove alignment padding from data pointer and length */
493  result->val.binary.data = (JsonbContainer *) (base_addr + INTALIGN(offset));
494  result->val.binary.len = getJsonbLength(container, index) -
495  (INTALIGN(offset) - offset);
496  }
497 }
498 
499 /*
500  * Push JsonbValue into JsonbParseState.
501  *
502  * Used when parsing JSON tokens to form Jsonb, or when converting an in-memory
503  * JsonbValue to a Jsonb.
504  *
505  * Initial state of *JsonbParseState is NULL, since it'll be allocated here
506  * originally (caller will get JsonbParseState back by reference).
507  *
508  * Only sequential tokens pertaining to non-container types should pass a
509  * JsonbValue. There is one exception -- WJB_BEGIN_ARRAY callers may pass a
510  * "raw scalar" pseudo array to append it - the actual scalar should be passed
511  * next and it will be added as the only member of the array.
512  *
513  * Values of type jvbBinary, which are rolled up arrays and objects,
514  * are unpacked before being added to the result.
515  */
516 JsonbValue *
518  JsonbValue *jbval)
519 {
520  JsonbIterator *it;
521  JsonbValue *res = NULL;
522  JsonbValue v;
523  JsonbIteratorToken tok;
524 
525  if (!jbval || (seq != WJB_ELEM && seq != WJB_VALUE) ||
526  jbval->type != jbvBinary)
527  {
528  /* drop through */
529  return pushJsonbValueScalar(pstate, seq, jbval);
530  }
531 
532  /* unpack the binary and add each piece to the pstate */
533  it = JsonbIteratorInit(jbval->val.binary.data);
534  while ((tok = JsonbIteratorNext(&it, &v, false)) != WJB_DONE)
535  res = pushJsonbValueScalar(pstate, tok,
536  tok < WJB_BEGIN_ARRAY ? &v : NULL);
537 
538  return res;
539 }
540 
541 /*
542  * Do the actual pushing, with only scalar or pseudo-scalar-array values
543  * accepted.
544  */
545 static JsonbValue *
547  JsonbValue *scalarVal)
548 {
550 
551  switch (seq)
552  {
553  case WJB_BEGIN_ARRAY:
554  Assert(!scalarVal || scalarVal->val.array.rawScalar);
555  *pstate = pushState(pstate);
556  result = &(*pstate)->contVal;
557  (*pstate)->contVal.type = jbvArray;
558  (*pstate)->contVal.val.array.nElems = 0;
559  (*pstate)->contVal.val.array.rawScalar = (scalarVal &&
560  scalarVal->val.array.rawScalar);
561  if (scalarVal && scalarVal->val.array.nElems > 0)
562  {
563  /* Assume that this array is still really a scalar */
564  Assert(scalarVal->type == jbvArray);
565  (*pstate)->size = scalarVal->val.array.nElems;
566  }
567  else
568  {
569  (*pstate)->size = 4;
570  }
571  (*pstate)->contVal.val.array.elems = palloc(sizeof(JsonbValue) *
572  (*pstate)->size);
573  break;
574  case WJB_BEGIN_OBJECT:
575  Assert(!scalarVal);
576  *pstate = pushState(pstate);
577  result = &(*pstate)->contVal;
578  (*pstate)->contVal.type = jbvObject;
579  (*pstate)->contVal.val.object.nPairs = 0;
580  (*pstate)->size = 4;
581  (*pstate)->contVal.val.object.pairs = palloc(sizeof(JsonbPair) *
582  (*pstate)->size);
583  break;
584  case WJB_KEY:
585  Assert(scalarVal->type == jbvString);
586  appendKey(*pstate, scalarVal);
587  break;
588  case WJB_VALUE:
589  Assert(IsAJsonbScalar(scalarVal));
590  appendValue(*pstate, scalarVal);
591  break;
592  case WJB_ELEM:
593  Assert(IsAJsonbScalar(scalarVal));
594  appendElement(*pstate, scalarVal);
595  break;
596  case WJB_END_OBJECT:
597  uniqueifyJsonbObject(&(*pstate)->contVal);
598  /* fall through! */
599  case WJB_END_ARRAY:
600  /* Steps here common to WJB_END_OBJECT case */
601  Assert(!scalarVal);
602  result = &(*pstate)->contVal;
603 
604  /*
605  * Pop stack and push current array/object as value in parent
606  * array/object
607  */
608  *pstate = (*pstate)->next;
609  if (*pstate)
610  {
611  switch ((*pstate)->contVal.type)
612  {
613  case jbvArray:
614  appendElement(*pstate, result);
615  break;
616  case jbvObject:
617  appendValue(*pstate, result);
618  break;
619  default:
620  elog(ERROR, "invalid jsonb container type");
621  }
622  }
623  break;
624  default:
625  elog(ERROR, "unrecognized jsonb sequential processing token");
626  }
627 
628  return result;
629 }
630 
631 /*
632  * pushJsonbValue() worker: Iteration-like forming of Jsonb
633  */
634 static JsonbParseState *
636 {
637  JsonbParseState *ns = palloc(sizeof(JsonbParseState));
638 
639  ns->next = *pstate;
640  return ns;
641 }
642 
643 /*
644  * pushJsonbValue() worker: Append a pair key to state when generating a Jsonb
645  */
646 static void
648 {
649  JsonbValue *object = &pstate->contVal;
650 
651  Assert(object->type == jbvObject);
652  Assert(string->type == jbvString);
653 
654  if (object->val.object.nPairs >= JSONB_MAX_PAIRS)
655  ereport(ERROR,
656  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
657  errmsg("number of jsonb object pairs exceeds the maximum allowed (%zu)",
658  JSONB_MAX_PAIRS)));
659 
660  if (object->val.object.nPairs >= pstate->size)
661  {
662  pstate->size *= 2;
663  object->val.object.pairs = repalloc(object->val.object.pairs,
664  sizeof(JsonbPair) * pstate->size);
665  }
666 
667  object->val.object.pairs[object->val.object.nPairs].key = *string;
668  object->val.object.pairs[object->val.object.nPairs].order = object->val.object.nPairs;
669 }
670 
671 /*
672  * pushJsonbValue() worker: Append a pair value to state when generating a
673  * Jsonb
674  */
675 static void
677 {
678  JsonbValue *object = &pstate->contVal;
679 
680  Assert(object->type == jbvObject);
681 
682  object->val.object.pairs[object->val.object.nPairs++].value = *scalarVal;
683 }
684 
685 /*
686  * pushJsonbValue() worker: Append an element to state when generating a Jsonb
687  */
688 static void
690 {
691  JsonbValue *array = &pstate->contVal;
692 
693  Assert(array->type == jbvArray);
694 
695  if (array->val.array.nElems >= JSONB_MAX_ELEMS)
696  ereport(ERROR,
697  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
698  errmsg("number of jsonb array elements exceeds the maximum allowed (%zu)",
699  JSONB_MAX_ELEMS)));
700 
701  if (array->val.array.nElems >= pstate->size)
702  {
703  pstate->size *= 2;
704  array->val.array.elems = repalloc(array->val.array.elems,
705  sizeof(JsonbValue) * pstate->size);
706  }
707 
708  array->val.array.elems[array->val.array.nElems++] = *scalarVal;
709 }
710 
711 /*
712  * Given a JsonbContainer, expand to JsonbIterator to iterate over items
713  * fully expanded to in-memory representation for manipulation.
714  *
715  * See JsonbIteratorNext() for notes on memory management.
716  */
719 {
720  return iteratorFromContainer(container, NULL);
721 }
722 
723 /*
724  * Get next JsonbValue while iterating
725  *
726  * Caller should initially pass their own, original iterator. They may get
727  * back a child iterator palloc()'d here instead. The function can be relied
728  * on to free those child iterators, lest the memory allocated for highly
729  * nested objects become unreasonable, but only if callers don't end iteration
730  * early (by breaking upon having found something in a search, for example).
731  *
732  * Callers in such a scenario, that are particularly sensitive to leaking
733  * memory in a long-lived context may walk the ancestral tree from the final
734  * iterator we left them with to its oldest ancestor, pfree()ing as they go.
735  * They do not have to free any other memory previously allocated for iterators
736  * but not accessible as direct ancestors of the iterator they're last passed
737  * back.
738  *
739  * Returns "Jsonb sequential processing" token value. Iterator "state"
740  * reflects the current stage of the process in a less granular fashion, and is
741  * mostly used here to track things internally with respect to particular
742  * iterators.
743  *
744  * Clients of this function should not have to handle any jbvBinary values
745  * (since recursive calls will deal with this), provided skipNested is false.
746  * It is our job to expand the jbvBinary representation without bothering them
747  * with it. However, clients should not take it upon themselves to touch array
748  * or Object element/pair buffers, since their element/pair pointers are
749  * garbage. Also, *val will not be set when returning WJB_END_ARRAY or
750  * WJB_END_OBJECT, on the assumption that it's only useful to access values
751  * when recursing in.
752  */
755 {
756  if (*it == NULL)
757  return WJB_DONE;
758 
759  /*
760  * When stepping into a nested container, we jump back here to start
761  * processing the child. We will not recurse further in one call, because
762  * processing the child will always begin in JBI_ARRAY_START or
763  * JBI_OBJECT_START state.
764  */
765 recurse:
766  switch ((*it)->state)
767  {
768  case JBI_ARRAY_START:
769  /* Set v to array on first array call */
770  val->type = jbvArray;
771  val->val.array.nElems = (*it)->nElems;
772 
773  /*
774  * v->val.array.elems is not actually set, because we aren't doing
775  * a full conversion
776  */
777  val->val.array.rawScalar = (*it)->isScalar;
778  (*it)->curIndex = 0;
779  (*it)->curDataOffset = 0;
780  (*it)->curValueOffset = 0; /* not actually used */
781  /* Set state for next call */
782  (*it)->state = JBI_ARRAY_ELEM;
783  return WJB_BEGIN_ARRAY;
784 
785  case JBI_ARRAY_ELEM:
786  if ((*it)->curIndex >= (*it)->nElems)
787  {
788  /*
789  * All elements within array already processed. Report this
790  * to caller, and give it back original parent iterator (which
791  * independently tracks iteration progress at its level of
792  * nesting).
793  */
794  *it = freeAndGetParent(*it);
795  return WJB_END_ARRAY;
796  }
797 
798  fillJsonbValue((*it)->container, (*it)->curIndex,
799  (*it)->dataProper, (*it)->curDataOffset,
800  val);
801 
802  JBE_ADVANCE_OFFSET((*it)->curDataOffset,
803  (*it)->children[(*it)->curIndex]);
804  (*it)->curIndex++;
805 
806  if (!IsAJsonbScalar(val) && !skipNested)
807  {
808  /* Recurse into container. */
809  *it = iteratorFromContainer(val->val.binary.data, *it);
810  goto recurse;
811  }
812  else
813  {
814  /*
815  * Scalar item in array, or a container and caller didn't want
816  * us to recurse into it.
817  */
818  return WJB_ELEM;
819  }
820 
821  case JBI_OBJECT_START:
822  /* Set v to object on first object call */
823  val->type = jbvObject;
824  val->val.object.nPairs = (*it)->nElems;
825 
826  /*
827  * v->val.object.pairs is not actually set, because we aren't
828  * doing a full conversion
829  */
830  (*it)->curIndex = 0;
831  (*it)->curDataOffset = 0;
832  (*it)->curValueOffset = getJsonbOffset((*it)->container,
833  (*it)->nElems);
834  /* Set state for next call */
835  (*it)->state = JBI_OBJECT_KEY;
836  return WJB_BEGIN_OBJECT;
837 
838  case JBI_OBJECT_KEY:
839  if ((*it)->curIndex >= (*it)->nElems)
840  {
841  /*
842  * All pairs within object already processed. Report this to
843  * caller, and give it back original containing iterator
844  * (which independently tracks iteration progress at its level
845  * of nesting).
846  */
847  *it = freeAndGetParent(*it);
848  return WJB_END_OBJECT;
849  }
850  else
851  {
852  /* Return key of a key/value pair. */
853  fillJsonbValue((*it)->container, (*it)->curIndex,
854  (*it)->dataProper, (*it)->curDataOffset,
855  val);
856  if (val->type != jbvString)
857  elog(ERROR, "unexpected jsonb type as object key");
858 
859  /* Set state for next call */
860  (*it)->state = JBI_OBJECT_VALUE;
861  return WJB_KEY;
862  }
863 
864  case JBI_OBJECT_VALUE:
865  /* Set state for next call */
866  (*it)->state = JBI_OBJECT_KEY;
867 
868  fillJsonbValue((*it)->container, (*it)->curIndex + (*it)->nElems,
869  (*it)->dataProper, (*it)->curValueOffset,
870  val);
871 
872  JBE_ADVANCE_OFFSET((*it)->curDataOffset,
873  (*it)->children[(*it)->curIndex]);
874  JBE_ADVANCE_OFFSET((*it)->curValueOffset,
875  (*it)->children[(*it)->curIndex + (*it)->nElems]);
876  (*it)->curIndex++;
877 
878  /*
879  * Value may be a container, in which case we recurse with new,
880  * child iterator (unless the caller asked not to, by passing
881  * skipNested).
882  */
883  if (!IsAJsonbScalar(val) && !skipNested)
884  {
885  *it = iteratorFromContainer(val->val.binary.data, *it);
886  goto recurse;
887  }
888  else
889  return WJB_VALUE;
890  }
891 
892  elog(ERROR, "invalid iterator state");
893  return -1;
894 }
895 
896 /*
897  * Initialize an iterator for iterating all elements in a container.
898  */
899 static JsonbIterator *
901 {
902  JsonbIterator *it;
903 
904  it = palloc(sizeof(JsonbIterator));
905  it->container = container;
906  it->parent = parent;
907  it->nElems = JsonContainerSize(container);
908 
909  /* Array starts just after header */
910  it->children = container->children;
911 
912  switch (container->header & (JB_FARRAY | JB_FOBJECT))
913  {
914  case JB_FARRAY:
915  it->dataProper =
916  (char *) it->children + it->nElems * sizeof(JEntry);
917  it->isScalar = JsonContainerIsScalar(container);
918  /* This is either a "raw scalar", or an array */
919  Assert(!it->isScalar || it->nElems == 1);
920 
921  it->state = JBI_ARRAY_START;
922  break;
923 
924  case JB_FOBJECT:
925  it->dataProper =
926  (char *) it->children + it->nElems * sizeof(JEntry) * 2;
927  it->state = JBI_OBJECT_START;
928  break;
929 
930  default:
931  elog(ERROR, "unknown type of jsonb container");
932  }
933 
934  return it;
935 }
936 
937 /*
938  * JsonbIteratorNext() worker: Return parent, while freeing memory for current
939  * iterator
940  */
941 static JsonbIterator *
943 {
944  JsonbIterator *v = it->parent;
945 
946  pfree(it);
947  return v;
948 }
949 
950 /*
951  * Worker for "contains" operator's function
952  *
953  * Formally speaking, containment is top-down, unordered subtree isomorphism.
954  *
955  * Takes iterators that belong to some container type. These iterators
956  * "belong" to those values in the sense that they've just been initialized in
957  * respect of them by the caller (perhaps in a nested fashion).
958  *
959  * "val" is lhs Jsonb, and mContained is rhs Jsonb when called from top level.
960  * We determine if mContained is contained within val.
961  */
962 bool
964 {
965  JsonbValue vval,
966  vcontained;
967  JsonbIteratorToken rval,
968  rcont;
969 
970  /*
971  * Guard against stack overflow due to overly complex Jsonb.
972  *
973  * Functions called here independently take this precaution, but that
974  * might not be sufficient since this is also a recursive function.
975  */
977 
978  rval = JsonbIteratorNext(val, &vval, false);
979  rcont = JsonbIteratorNext(mContained, &vcontained, false);
980 
981  if (rval != rcont)
982  {
983  /*
984  * The differing return values can immediately be taken as indicating
985  * two differing container types at this nesting level, which is
986  * sufficient reason to give up entirely (but it should be the case
987  * that they're both some container type).
988  */
989  Assert(rval == WJB_BEGIN_OBJECT || rval == WJB_BEGIN_ARRAY);
990  Assert(rcont == WJB_BEGIN_OBJECT || rcont == WJB_BEGIN_ARRAY);
991  return false;
992  }
993  else if (rcont == WJB_BEGIN_OBJECT)
994  {
995  Assert(vval.type == jbvObject);
996  Assert(vcontained.type == jbvObject);
997 
998  /*
999  * If the lhs has fewer pairs than the rhs, it can't possibly contain
1000  * the rhs. (This conclusion is safe only because we de-duplicate
1001  * keys in all Jsonb objects; thus there can be no corresponding
1002  * optimization in the array case.) The case probably won't arise
1003  * often, but since it's such a cheap check we may as well make it.
1004  */
1005  if (vval.val.object.nPairs < vcontained.val.object.nPairs)
1006  return false;
1007 
1008  /* Work through rhs "is it contained within?" object */
1009  for (;;)
1010  {
1011  JsonbValue *lhsVal; /* lhsVal is from pair in lhs object */
1012 
1013  rcont = JsonbIteratorNext(mContained, &vcontained, false);
1014 
1015  /*
1016  * When we get through caller's rhs "is it contained within?"
1017  * object without failing to find one of its values, it's
1018  * contained.
1019  */
1020  if (rcont == WJB_END_OBJECT)
1021  return true;
1022 
1023  Assert(rcont == WJB_KEY);
1024 
1025  /* First, find value by key... */
1026  lhsVal = findJsonbValueFromContainer((*val)->container,
1027  JB_FOBJECT,
1028  &vcontained);
1029 
1030  if (!lhsVal)
1031  return false;
1032 
1033  /*
1034  * ...at this stage it is apparent that there is at least a key
1035  * match for this rhs pair.
1036  */
1037  rcont = JsonbIteratorNext(mContained, &vcontained, true);
1038 
1039  Assert(rcont == WJB_VALUE);
1040 
1041  /*
1042  * Compare rhs pair's value with lhs pair's value just found using
1043  * key
1044  */
1045  if (lhsVal->type != vcontained.type)
1046  {
1047  return false;
1048  }
1049  else if (IsAJsonbScalar(lhsVal))
1050  {
1051  if (!equalsJsonbScalarValue(lhsVal, &vcontained))
1052  return false;
1053  }
1054  else
1055  {
1056  /* Nested container value (object or array) */
1057  JsonbIterator *nestval,
1058  *nestContained;
1059 
1060  Assert(lhsVal->type == jbvBinary);
1061  Assert(vcontained.type == jbvBinary);
1062 
1063  nestval = JsonbIteratorInit(lhsVal->val.binary.data);
1064  nestContained = JsonbIteratorInit(vcontained.val.binary.data);
1065 
1066  /*
1067  * Match "value" side of rhs datum object's pair recursively.
1068  * It's a nested structure.
1069  *
1070  * Note that nesting still has to "match up" at the right
1071  * nesting sub-levels. However, there need only be zero or
1072  * more matching pairs (or elements) at each nesting level
1073  * (provided the *rhs* pairs/elements *all* match on each
1074  * level), which enables searching nested structures for a
1075  * single String or other primitive type sub-datum quite
1076  * effectively (provided the user constructed the rhs nested
1077  * structure such that we "know where to look").
1078  *
1079  * In other words, the mapping of container nodes in the rhs
1080  * "vcontained" Jsonb to internal nodes on the lhs is
1081  * injective, and parent-child edges on the rhs must be mapped
1082  * to parent-child edges on the lhs to satisfy the condition
1083  * of containment (plus of course the mapped nodes must be
1084  * equal).
1085  */
1086  if (!JsonbDeepContains(&nestval, &nestContained))
1087  return false;
1088  }
1089  }
1090  }
1091  else if (rcont == WJB_BEGIN_ARRAY)
1092  {
1093  JsonbValue *lhsConts = NULL;
1094  uint32 nLhsElems = vval.val.array.nElems;
1095 
1096  Assert(vval.type == jbvArray);
1097  Assert(vcontained.type == jbvArray);
1098 
1099  /*
1100  * Handle distinction between "raw scalar" pseudo arrays, and real
1101  * arrays.
1102  *
1103  * A raw scalar may contain another raw scalar, and an array may
1104  * contain a raw scalar, but a raw scalar may not contain an array. We
1105  * don't do something like this for the object case, since objects can
1106  * only contain pairs, never raw scalars (a pair is represented by an
1107  * rhs object argument with a single contained pair).
1108  */
1109  if (vval.val.array.rawScalar && !vcontained.val.array.rawScalar)
1110  return false;
1111 
1112  /* Work through rhs "is it contained within?" array */
1113  for (;;)
1114  {
1115  rcont = JsonbIteratorNext(mContained, &vcontained, true);
1116 
1117  /*
1118  * When we get through caller's rhs "is it contained within?"
1119  * array without failing to find one of its values, it's
1120  * contained.
1121  */
1122  if (rcont == WJB_END_ARRAY)
1123  return true;
1124 
1125  Assert(rcont == WJB_ELEM);
1126 
1127  if (IsAJsonbScalar(&vcontained))
1128  {
1129  if (!findJsonbValueFromContainer((*val)->container,
1130  JB_FARRAY,
1131  &vcontained))
1132  return false;
1133  }
1134  else
1135  {
1136  uint32 i;
1137 
1138  /*
1139  * If this is first container found in rhs array (at this
1140  * depth), initialize temp lhs array of containers
1141  */
1142  if (lhsConts == NULL)
1143  {
1144  uint32 j = 0;
1145 
1146  /* Make room for all possible values */
1147  lhsConts = palloc(sizeof(JsonbValue) * nLhsElems);
1148 
1149  for (i = 0; i < nLhsElems; i++)
1150  {
1151  /* Store all lhs elements in temp array */
1152  rcont = JsonbIteratorNext(val, &vval, true);
1153  Assert(rcont == WJB_ELEM);
1154 
1155  if (vval.type == jbvBinary)
1156  lhsConts[j++] = vval;
1157  }
1158 
1159  /* No container elements in temp array, so give up now */
1160  if (j == 0)
1161  return false;
1162 
1163  /* We may have only partially filled array */
1164  nLhsElems = j;
1165  }
1166 
1167  /* XXX: Nested array containment is O(N^2) */
1168  for (i = 0; i < nLhsElems; i++)
1169  {
1170  /* Nested container value (object or array) */
1171  JsonbIterator *nestval,
1172  *nestContained;
1173  bool contains;
1174 
1175  nestval = JsonbIteratorInit(lhsConts[i].val.binary.data);
1176  nestContained = JsonbIteratorInit(vcontained.val.binary.data);
1177 
1178  contains = JsonbDeepContains(&nestval, &nestContained);
1179 
1180  if (nestval)
1181  pfree(nestval);
1182  if (nestContained)
1183  pfree(nestContained);
1184  if (contains)
1185  break;
1186  }
1187 
1188  /*
1189  * Report rhs container value is not contained if couldn't
1190  * match rhs container to *some* lhs cont
1191  */
1192  if (i == nLhsElems)
1193  return false;
1194  }
1195  }
1196  }
1197  else
1198  {
1199  elog(ERROR, "invalid jsonb container type");
1200  }
1201 
1202  elog(ERROR, "unexpectedly fell off end of jsonb container");
1203  return false;
1204 }
1205 
1206 /*
1207  * Hash a JsonbValue scalar value, mixing the hash value into an existing
1208  * hash provided by the caller.
1209  *
1210  * Some callers may wish to independently XOR in JB_FOBJECT and JB_FARRAY
1211  * flags.
1212  */
1213 void
1215 {
1216  uint32 tmp;
1217 
1218  /* Compute hash value for scalarVal */
1219  switch (scalarVal->type)
1220  {
1221  case jbvNull:
1222  tmp = 0x01;
1223  break;
1224  case jbvString:
1225  tmp = DatumGetUInt32(hash_any((const unsigned char *) scalarVal->val.string.val,
1226  scalarVal->val.string.len));
1227  break;
1228  case jbvNumeric:
1229  /* Must hash equal numerics to equal hash codes */
1231  NumericGetDatum(scalarVal->val.numeric)));
1232  break;
1233  case jbvBool:
1234  tmp = scalarVal->val.boolean ? 0x02 : 0x04;
1235 
1236  break;
1237  default:
1238  elog(ERROR, "invalid jsonb scalar type");
1239  tmp = 0; /* keep compiler quiet */
1240  break;
1241  }
1242 
1243  /*
1244  * Combine hash values of successive keys, values and elements by rotating
1245  * the previous value left 1 bit, then XOR'ing in the new
1246  * key/value/element's hash value.
1247  */
1248  *hash = (*hash << 1) | (*hash >> 31);
1249  *hash ^= tmp;
1250 }
1251 
1252 /*
1253  * Are two scalar JsonbValues of the same type a and b equal?
1254  */
1255 static bool
1257 {
1258  if (aScalar->type == bScalar->type)
1259  {
1260  switch (aScalar->type)
1261  {
1262  case jbvNull:
1263  return true;
1264  case jbvString:
1265  return lengthCompareJsonbStringValue(aScalar, bScalar) == 0;
1266  case jbvNumeric:
1268  PointerGetDatum(aScalar->val.numeric),
1269  PointerGetDatum(bScalar->val.numeric)));
1270  case jbvBool:
1271  return aScalar->val.boolean == bScalar->val.boolean;
1272 
1273  default:
1274  elog(ERROR, "invalid jsonb scalar type");
1275  }
1276  }
1277  elog(ERROR, "jsonb scalar type mismatch");
1278  return -1;
1279 }
1280 
1281 /*
1282  * Compare two scalar JsonbValues, returning -1, 0, or 1.
1283  *
1284  * Strings are compared using the default collation. Used by B-tree
1285  * operators, where a lexical sort order is generally expected.
1286  */
1287 static int
1289 {
1290  if (aScalar->type == bScalar->type)
1291  {
1292  switch (aScalar->type)
1293  {
1294  case jbvNull:
1295  return 0;
1296  case jbvString:
1297  return varstr_cmp(aScalar->val.string.val,
1298  aScalar->val.string.len,
1299  bScalar->val.string.val,
1300  bScalar->val.string.len,
1302  case jbvNumeric:
1304  PointerGetDatum(aScalar->val.numeric),
1305  PointerGetDatum(bScalar->val.numeric)));
1306  case jbvBool:
1307  if (aScalar->val.boolean == bScalar->val.boolean)
1308  return 0;
1309  else if (aScalar->val.boolean > bScalar->val.boolean)
1310  return 1;
1311  else
1312  return -1;
1313  default:
1314  elog(ERROR, "invalid jsonb scalar type");
1315  }
1316  }
1317  elog(ERROR, "jsonb scalar type mismatch");
1318  return -1;
1319 }
1320 
1321 
1322 /*
1323  * Functions for manipulating the resizeable buffer used by convertJsonb and
1324  * its subroutines.
1325  */
1326 
1327 /*
1328  * Reserve 'len' bytes, at the end of the buffer, enlarging it if necessary.
1329  * Returns the offset to the reserved area. The caller is expected to fill
1330  * the reserved area later with copyToBuffer().
1331  */
1332 static int
1334 {
1335  int offset;
1336 
1337  /* Make more room if needed */
1338  enlargeStringInfo(buffer, len);
1339 
1340  /* remember current offset */
1341  offset = buffer->len;
1342 
1343  /* reserve the space */
1344  buffer->len += len;
1345 
1346  /*
1347  * Keep a trailing null in place, even though it's not useful for us; it
1348  * seems best to preserve the invariants of StringInfos.
1349  */
1350  buffer->data[buffer->len] = '\0';
1351 
1352  return offset;
1353 }
1354 
1355 /*
1356  * Copy 'len' bytes to a previously reserved area in buffer.
1357  */
1358 static void
1359 copyToBuffer(StringInfo buffer, int offset, const char *data, int len)
1360 {
1361  memcpy(buffer->data + offset, data, len);
1362 }
1363 
1364 /*
1365  * A shorthand for reserveFromBuffer + copyToBuffer.
1366  */
1367 static void
1368 appendToBuffer(StringInfo buffer, const char *data, int len)
1369 {
1370  int offset;
1371 
1372  offset = reserveFromBuffer(buffer, len);
1373  copyToBuffer(buffer, offset, data, len);
1374 }
1375 
1376 
1377 /*
1378  * Append padding, so that the length of the StringInfo is int-aligned.
1379  * Returns the number of padding bytes appended.
1380  */
1381 static short
1383 {
1384  int padlen,
1385  p,
1386  offset;
1387 
1388  padlen = INTALIGN(buffer->len) - buffer->len;
1389 
1390  offset = reserveFromBuffer(buffer, padlen);
1391 
1392  /* padlen must be small, so this is probably faster than a memset */
1393  for (p = 0; p < padlen; p++)
1394  buffer->data[offset + p] = '\0';
1395 
1396  return padlen;
1397 }
1398 
1399 /*
1400  * Given a JsonbValue, convert to Jsonb. The result is palloc'd.
1401  */
1402 static Jsonb *
1404 {
1406  JEntry jentry;
1407  Jsonb *res;
1408 
1409  /* Should not already have binary representation */
1410  Assert(val->type != jbvBinary);
1411 
1412  /* Allocate an output buffer. It will be enlarged as needed */
1413  initStringInfo(&buffer);
1414 
1415  /* Make room for the varlena header */
1416  reserveFromBuffer(&buffer, VARHDRSZ);
1417 
1418  convertJsonbValue(&buffer, &jentry, val, 0);
1419 
1420  /*
1421  * Note: the JEntry of the root is discarded. Therefore the root
1422  * JsonbContainer struct must contain enough information to tell what kind
1423  * of value it is.
1424  */
1425 
1426  res = (Jsonb *) buffer.data;
1427 
1428  SET_VARSIZE(res, buffer.len);
1429 
1430  return res;
1431 }
1432 
1433 /*
1434  * Subroutine of convertJsonb: serialize a single JsonbValue into buffer.
1435  *
1436  * The JEntry header for this node is returned in *header. It is filled in
1437  * with the length of this value and appropriate type bits. If we wish to
1438  * store an end offset rather than a length, it is the caller's responsibility
1439  * to adjust for that.
1440  *
1441  * If the value is an array or an object, this recurses. 'level' is only used
1442  * for debugging purposes.
1443  */
1444 static void
1446 {
1448 
1449  if (!val)
1450  return;
1451 
1452  /*
1453  * A JsonbValue passed as val should never have a type of jbvBinary, and
1454  * neither should any of its sub-components. Those values will be produced
1455  * by convertJsonbArray and convertJsonbObject, the results of which will
1456  * not be passed back to this function as an argument.
1457  */
1458 
1459  if (IsAJsonbScalar(val))
1460  convertJsonbScalar(buffer, header, val);
1461  else if (val->type == jbvArray)
1462  convertJsonbArray(buffer, header, val, level);
1463  else if (val->type == jbvObject)
1464  convertJsonbObject(buffer, header, val, level);
1465  else
1466  elog(ERROR, "unknown type of jsonb container to convert");
1467 }
1468 
1469 static void
1471 {
1472  int base_offset;
1473  int jentry_offset;
1474  int i;
1475  int totallen;
1476  uint32 header;
1477  int nElems = val->val.array.nElems;
1478 
1479  /* Remember where in the buffer this array starts. */
1480  base_offset = buffer->len;
1481 
1482  /* Align to 4-byte boundary (any padding counts as part of my data) */
1483  padBufferToInt(buffer);
1484 
1485  /*
1486  * Construct the header Jentry and store it in the beginning of the
1487  * variable-length payload.
1488  */
1489  header = nElems | JB_FARRAY;
1490  if (val->val.array.rawScalar)
1491  {
1492  Assert(nElems == 1);
1493  Assert(level == 0);
1494  header |= JB_FSCALAR;
1495  }
1496 
1497  appendToBuffer(buffer, (char *) &header, sizeof(uint32));
1498 
1499  /* Reserve space for the JEntries of the elements. */
1500  jentry_offset = reserveFromBuffer(buffer, sizeof(JEntry) * nElems);
1501 
1502  totallen = 0;
1503  for (i = 0; i < nElems; i++)
1504  {
1505  JsonbValue *elem = &val->val.array.elems[i];
1506  int len;
1507  JEntry meta;
1508 
1509  /*
1510  * Convert element, producing a JEntry and appending its
1511  * variable-length data to buffer
1512  */
1513  convertJsonbValue(buffer, &meta, elem, level + 1);
1514 
1515  len = JBE_OFFLENFLD(meta);
1516  totallen += len;
1517 
1518  /*
1519  * Bail out if total variable-length data exceeds what will fit in a
1520  * JEntry length field. We check this in each iteration, not just
1521  * once at the end, to forestall possible integer overflow.
1522  */
1523  if (totallen > JENTRY_OFFLENMASK)
1524  ereport(ERROR,
1525  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1526  errmsg("total size of jsonb array elements exceeds the maximum of %u bytes",
1527  JENTRY_OFFLENMASK)));
1528 
1529  /*
1530  * Convert each JB_OFFSET_STRIDE'th length to an offset.
1531  */
1532  if ((i % JB_OFFSET_STRIDE) == 0)
1533  meta = (meta & JENTRY_TYPEMASK) | totallen | JENTRY_HAS_OFF;
1534 
1535  copyToBuffer(buffer, jentry_offset, (char *) &meta, sizeof(JEntry));
1536  jentry_offset += sizeof(JEntry);
1537  }
1538 
1539  /* Total data size is everything we've appended to buffer */
1540  totallen = buffer->len - base_offset;
1541 
1542  /* Check length again, since we didn't include the metadata above */
1543  if (totallen > JENTRY_OFFLENMASK)
1544  ereport(ERROR,
1545  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1546  errmsg("total size of jsonb array elements exceeds the maximum of %u bytes",
1547  JENTRY_OFFLENMASK)));
1548 
1549  /* Initialize the header of this node in the container's JEntry array */
1550  *pheader = JENTRY_ISCONTAINER | totallen;
1551 }
1552 
1553 static void
1555 {
1556  int base_offset;
1557  int jentry_offset;
1558  int i;
1559  int totallen;
1560  uint32 header;
1561  int nPairs = val->val.object.nPairs;
1562 
1563  /* Remember where in the buffer this object starts. */
1564  base_offset = buffer->len;
1565 
1566  /* Align to 4-byte boundary (any padding counts as part of my data) */
1567  padBufferToInt(buffer);
1568 
1569  /*
1570  * Construct the header Jentry and store it in the beginning of the
1571  * variable-length payload.
1572  */
1573  header = nPairs | JB_FOBJECT;
1574  appendToBuffer(buffer, (char *) &header, sizeof(uint32));
1575 
1576  /* Reserve space for the JEntries of the keys and values. */
1577  jentry_offset = reserveFromBuffer(buffer, sizeof(JEntry) * nPairs * 2);
1578 
1579  /*
1580  * Iterate over the keys, then over the values, since that is the ordering
1581  * we want in the on-disk representation.
1582  */
1583  totallen = 0;
1584  for (i = 0; i < nPairs; i++)
1585  {
1586  JsonbPair *pair = &val->val.object.pairs[i];
1587  int len;
1588  JEntry meta;
1589 
1590  /*
1591  * Convert key, producing a JEntry and appending its variable-length
1592  * data to buffer
1593  */
1594  convertJsonbScalar(buffer, &meta, &pair->key);
1595 
1596  len = JBE_OFFLENFLD(meta);
1597  totallen += len;
1598 
1599  /*
1600  * Bail out if total variable-length data exceeds what will fit in a
1601  * JEntry length field. We check this in each iteration, not just
1602  * once at the end, to forestall possible integer overflow.
1603  */
1604  if (totallen > JENTRY_OFFLENMASK)
1605  ereport(ERROR,
1606  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1607  errmsg("total size of jsonb object elements exceeds the maximum of %u bytes",
1608  JENTRY_OFFLENMASK)));
1609 
1610  /*
1611  * Convert each JB_OFFSET_STRIDE'th length to an offset.
1612  */
1613  if ((i % JB_OFFSET_STRIDE) == 0)
1614  meta = (meta & JENTRY_TYPEMASK) | totallen | JENTRY_HAS_OFF;
1615 
1616  copyToBuffer(buffer, jentry_offset, (char *) &meta, sizeof(JEntry));
1617  jentry_offset += sizeof(JEntry);
1618  }
1619  for (i = 0; i < nPairs; i++)
1620  {
1621  JsonbPair *pair = &val->val.object.pairs[i];
1622  int len;
1623  JEntry meta;
1624 
1625  /*
1626  * Convert value, producing a JEntry and appending its variable-length
1627  * data to buffer
1628  */
1629  convertJsonbValue(buffer, &meta, &pair->value, level + 1);
1630 
1631  len = JBE_OFFLENFLD(meta);
1632  totallen += len;
1633 
1634  /*
1635  * Bail out if total variable-length data exceeds what will fit in a
1636  * JEntry length field. We check this in each iteration, not just
1637  * once at the end, to forestall possible integer overflow.
1638  */
1639  if (totallen > JENTRY_OFFLENMASK)
1640  ereport(ERROR,
1641  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1642  errmsg("total size of jsonb object elements exceeds the maximum of %u bytes",
1643  JENTRY_OFFLENMASK)));
1644 
1645  /*
1646  * Convert each JB_OFFSET_STRIDE'th length to an offset.
1647  */
1648  if (((i + nPairs) % JB_OFFSET_STRIDE) == 0)
1649  meta = (meta & JENTRY_TYPEMASK) | totallen | JENTRY_HAS_OFF;
1650 
1651  copyToBuffer(buffer, jentry_offset, (char *) &meta, sizeof(JEntry));
1652  jentry_offset += sizeof(JEntry);
1653  }
1654 
1655  /* Total data size is everything we've appended to buffer */
1656  totallen = buffer->len - base_offset;
1657 
1658  /* Check length again, since we didn't include the metadata above */
1659  if (totallen > JENTRY_OFFLENMASK)
1660  ereport(ERROR,
1661  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1662  errmsg("total size of jsonb object elements exceeds the maximum of %u bytes",
1663  JENTRY_OFFLENMASK)));
1664 
1665  /* Initialize the header of this node in the container's JEntry array */
1666  *pheader = JENTRY_ISCONTAINER | totallen;
1667 }
1668 
1669 static void
1671 {
1672  int numlen;
1673  short padlen;
1674 
1675  switch (scalarVal->type)
1676  {
1677  case jbvNull:
1678  *jentry = JENTRY_ISNULL;
1679  break;
1680 
1681  case jbvString:
1682  appendToBuffer(buffer, scalarVal->val.string.val, scalarVal->val.string.len);
1683 
1684  *jentry = scalarVal->val.string.len;
1685  break;
1686 
1687  case jbvNumeric:
1688  numlen = VARSIZE_ANY(scalarVal->val.numeric);
1689  padlen = padBufferToInt(buffer);
1690 
1691  appendToBuffer(buffer, (char *) scalarVal->val.numeric, numlen);
1692 
1693  *jentry = JENTRY_ISNUMERIC | (padlen + numlen);
1694  break;
1695 
1696  case jbvBool:
1697  *jentry = (scalarVal->val.boolean) ?
1699  break;
1700 
1701  default:
1702  elog(ERROR, "invalid jsonb scalar type");
1703  }
1704 }
1705 
1706 /*
1707  * Compare two jbvString JsonbValue values, a and b.
1708  *
1709  * This is a special qsort() comparator used to sort strings in certain
1710  * internal contexts where it is sufficient to have a well-defined sort order.
1711  * In particular, object pair keys are sorted according to this criteria to
1712  * facilitate cheap binary searches where we don't care about lexical sort
1713  * order.
1714  *
1715  * a and b are first sorted based on their length. If a tie-breaker is
1716  * required, only then do we consider string binary equality.
1717  */
1718 static int
1719 lengthCompareJsonbStringValue(const void *a, const void *b)
1720 {
1721  const JsonbValue *va = (const JsonbValue *) a;
1722  const JsonbValue *vb = (const JsonbValue *) b;
1723  int res;
1724 
1725  Assert(va->type == jbvString);
1726  Assert(vb->type == jbvString);
1727 
1728  if (va->val.string.len == vb->val.string.len)
1729  {
1730  res = memcmp(va->val.string.val, vb->val.string.val, va->val.string.len);
1731  }
1732  else
1733  {
1734  res = (va->val.string.len > vb->val.string.len) ? 1 : -1;
1735  }
1736 
1737  return res;
1738 }
1739 
1740 /*
1741  * qsort_arg() comparator to compare JsonbPair values.
1742  *
1743  * Third argument 'binequal' may point to a bool. If it's set, *binequal is set
1744  * to true iff a and b have full binary equality, since some callers have an
1745  * interest in whether the two values are equal or merely equivalent.
1746  *
1747  * N.B: String comparisons here are "length-wise"
1748  *
1749  * Pairs with equals keys are ordered such that the order field is respected.
1750  */
1751 static int
1752 lengthCompareJsonbPair(const void *a, const void *b, void *binequal)
1753 {
1754  const JsonbPair *pa = (const JsonbPair *) a;
1755  const JsonbPair *pb = (const JsonbPair *) b;
1756  int res;
1757 
1758  res = lengthCompareJsonbStringValue(&pa->key, &pb->key);
1759  if (res == 0 && binequal)
1760  *((bool *) binequal) = true;
1761 
1762  /*
1763  * Guarantee keeping order of equal pair. Unique algorithm will prefer
1764  * first element as value.
1765  */
1766  if (res == 0)
1767  res = (pa->order > pb->order) ? -1 : 1;
1768 
1769  return res;
1770 }
1771 
1772 /*
1773  * Sort and unique-ify pairs in JsonbValue object
1774  */
1775 static void
1777 {
1778  bool hasNonUniq = false;
1779 
1780  Assert(object->type == jbvObject);
1781 
1782  if (object->val.object.nPairs > 1)
1783  qsort_arg(object->val.object.pairs, object->val.object.nPairs, sizeof(JsonbPair),
1784  lengthCompareJsonbPair, &hasNonUniq);
1785 
1786  if (hasNonUniq)
1787  {
1788  JsonbPair *ptr = object->val.object.pairs + 1,
1789  *res = object->val.object.pairs;
1790 
1791  while (ptr - object->val.object.pairs < object->val.object.nPairs)
1792  {
1793  /* Avoid copying over duplicate */
1794  if (lengthCompareJsonbStringValue(ptr, res) != 0)
1795  {
1796  res++;
1797  if (ptr != res)
1798  memcpy(res, ptr, sizeof(JsonbPair));
1799  }
1800  ptr++;
1801  }
1802 
1803  object->val.object.nPairs = res + 1 - object->val.object.pairs;
1804  }
1805 }
#define DatumGetUInt32(X)
Definition: postgres.h:492
JsonbIterState state
Definition: jsonb.h:349
uint32 order
Definition: jsonb.h:300
JEntry children[FLEXIBLE_ARRAY_MEMBER]
Definition: jsonb.h:197
struct PathHashStack * parent
Definition: jsonb_gin.c:28
static void appendValue(JsonbParseState *pstate, JsonbValue *scalarVal)
Definition: jsonb_util.c:676
#define JENTRY_ISNULL
Definition: jsonb.h:150
uint32 getJsonbOffset(const JsonbContainer *jc, int index)
Definition: jsonb_util.c:121
#define JB_FARRAY
Definition: jsonb.h:206
JEntry * children
Definition: jsonb.h:331
#define VARDATA(PTR)
Definition: postgres.h:303
Jsonb * JsonbValueToJsonb(JsonbValue *val)
Definition: jsonb_util.c:79
Definition: jsonb.h:215
#define JBE_ISCONTAINER(je_)
Definition: jsonb.h:158
#define DatumGetInt32(X)
Definition: postgres.h:478
Datum hash_numeric(PG_FUNCTION_ARGS)
Definition: numeric.c:2158
bool JsonbDeepContains(JsonbIterator **val, JsonbIterator **mContained)
Definition: jsonb_util.c:963
#define PointerGetDatum(X)
Definition: postgres.h:562
#define VARHDRSZ
Definition: c.h:445
static void convertJsonbValue(StringInfo buffer, JEntry *header, JsonbValue *val, int level)
Definition: jsonb_util.c:1445
char * val
Definition: jsonb.h:259
#define NumericGetDatum(X)
Definition: numeric.h:51
#define JBE_ISBOOL_TRUE(je_)
Definition: jsonb.h:160
static void appendToBuffer(StringInfo buffer, const char *data, int len)
Definition: jsonb_util.c:1368
Definition: jsonb.h:234
#define JsonContainerIsScalar(jc)
Definition: jsonb.h:210
Datum numeric_cmp(PG_FUNCTION_ARGS)
Definition: numeric.c:2020
Definition: jsonb.h:22
uint32 getJsonbLength(const JsonbContainer *jc, int index)
Definition: jsonb_util.c:146
static JsonbParseState * pushState(JsonbParseState **pstate)
Definition: jsonb_util.c:635
int errcode(int sqlerrcode)
Definition: elog.c:575
return result
Definition: formatting.c:1618
#define JENTRY_OFFLENMASK
Definition: jsonb.h:141
Definition: jsonb.h:231
#define INTALIGN(LEN)
Definition: c.h:585
#define JSONB_MAX_PAIRS
Definition: jsonb_util.c:34
static JsonbIterator * freeAndGetParent(JsonbIterator *it)
Definition: jsonb_util.c:942
static void appendElement(JsonbParseState *pstate, JsonbValue *scalarVal)
Definition: jsonb_util.c:689
#define DirectFunctionCall1(func, arg1)
Definition: fmgr.h:584
struct NumericData * Numeric
Definition: numeric.h:43
#define JBE_OFFLENFLD(je_)
Definition: jsonb.h:154
struct JsonbParseState * next
Definition: jsonb.h:308
#define JBE_ISSTRING(je_)
Definition: jsonb.h:156
#define JB_OFFSET_STRIDE
Definition: jsonb.h:181
#define JENTRY_ISNUMERIC
Definition: jsonb.h:147
Datum numeric_eq(PG_FUNCTION_ARGS)
Definition: numeric.c:2036
Definition: type.h:90
#define JBE_ADVANCE_OFFSET(offset, je)
Definition: jsonb.h:165
JsonbValue * pushJsonbValue(JsonbParseState **pstate, JsonbIteratorToken seq, JsonbValue *jbval)
Definition: jsonb_util.c:517
#define JSONB_MAX_ELEMS
Definition: jsonb_util.c:33
void JsonbHashScalarValue(const JsonbValue *scalarVal, uint32 *hash)
Definition: jsonb_util.c:1214
void pfree(void *pointer)
Definition: mcxt.c:950
#define ERROR
Definition: elog.h:43
static int lengthCompareJsonbPair(const void *a, const void *b, void *arg)
Definition: jsonb_util.c:1752
#define JB_FSCALAR
Definition: jsonb.h:204
#define JENTRY_TYPEMASK
Definition: jsonb.h:142
JsonbContainer * container
Definition: jsonb.h:327
#define DEFAULT_COLLATION_OID
Definition: pg_collation.h:74
void check_stack_depth(void)
Definition: postgres.c:3102
static int reserveFromBuffer(StringInfo buffer, int len)
Definition: jsonb_util.c:1333
char string[11]
Definition: preproc-type.c:46
#define DatumGetBool(X)
Definition: postgres.h:399
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:277
#define JsonContainerSize(jc)
Definition: jsonb.h:209
unsigned int uint32
Definition: c.h:268
JsonbValue key
Definition: jsonb.h:298
Definition: jsonb.h:23
int compareJsonbContainers(JsonbContainer *a, JsonbContainer *b)
Definition: jsonb_util.c:178
static void fillJsonbValue(JsonbContainer *container, int index, char *base_addr, uint32 offset, JsonbValue *result)
Definition: jsonb_util.c:456
#define JsonContainerIsArray(jc)
Definition: jsonb.h:212
static JsonbValue * pushJsonbValueScalar(JsonbParseState **pstate, JsonbIteratorToken seq, JsonbValue *scalarVal)
Definition: jsonb_util.c:546
#define ereport(elevel, rest)
Definition: elog.h:122
static void uniqueifyJsonbObject(JsonbValue *object)
Definition: jsonb_util.c:1776
static bool equalsJsonbScalarValue(JsonbValue *a, JsonbValue *b)
Definition: jsonb_util.c:1256
JsonbIteratorToken
Definition: jsonb.h:20
static void convertJsonbArray(StringInfo buffer, JEntry *header, JsonbValue *val, int level)
Definition: jsonb_util.c:1470
void initStringInfo(StringInfo str)
Definition: stringinfo.c:65
void qsort_arg(void *base, size_t nel, size_t elsize, qsort_arg_comparator cmp, void *arg)
Definition: qsort_arg.c:113
static void convertJsonbScalar(StringInfo buffer, JEntry *header, JsonbValue *scalarVal)
Definition: jsonb_util.c:1670
uint32 header
Definition: jsonb.h:195
static short padBufferToInt(StringInfo buffer)
Definition: jsonb_util.c:1382
#define JsonContainerIsObject(jc)
Definition: jsonb.h:211
Datum difference(PG_FUNCTION_ARGS)
static void copyToBuffer(StringInfo buffer, int offset, const char *data, int len)
Definition: jsonb_util.c:1359
JsonbValue contVal
Definition: jsonb.h:306
#define VARSIZE_ANY(PTR)
Definition: postgres.h:334
bool isScalar
Definition: jsonb.h:330
#define JENTRY_ISCONTAINER
Definition: jsonb.h:151
static int compareJsonbScalarValue(JsonbValue *a, JsonbValue *b)
Definition: jsonb_util.c:1288
JsonbValue * getIthJsonbValueFromContainer(JsonbContainer *container, uint32 i)
Definition: jsonb_util.c:419
static JsonbIterator * iteratorFromContainer(JsonbContainer *container, JsonbIterator *parent)
Definition: jsonb_util.c:900
#define NULL
Definition: c.h:229
#define Assert(condition)
Definition: c.h:675
JsonbIterator * JsonbIteratorInit(JsonbContainer *container)
Definition: jsonb_util.c:718
#define JBE_ISNUMERIC(je_)
Definition: jsonb.h:157
#define JB_FOBJECT
Definition: jsonb.h:205
struct JsonbIterator * parent
Definition: jsonb.h:351
Datum hash_any(register const unsigned char *k, register int keylen)
Definition: hashfunc.c:307
int varstr_cmp(char *arg1, int len1, char *arg2, int len2, Oid collid)
Definition: varlena.c:1382
uint32 JEntry
Definition: jsonb.h:139
WalTimeSample buffer[LAG_TRACKER_BUFFER_SIZE]
Definition: walsender.c:207
static Jsonb * convertToJsonb(JsonbValue *val)
Definition: jsonb_util.c:1403
#define JBE_ISNULL(je_)
Definition: jsonb.h:159
#define JENTRY_HAS_OFF
Definition: jsonb.h:143
char * dataProper
Definition: jsonb.h:333
#define IsAJsonbScalar(jsonbval)
Definition: jsonb.h:283
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:207
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:963
JsonbValue value
Definition: jsonb.h:299
enum jbvType type
Definition: jsonb.h:250
void * palloc(Size size)
Definition: mcxt.c:849
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define JENTRY_ISBOOL_TRUE
Definition: jsonb.h:149
#define JBE_HAS_OFF(je_)
Definition: jsonb.h:155
#define JENTRY_ISBOOL_FALSE
Definition: jsonb.h:148
int i
#define JBE_ISBOOL_FALSE(je_)
Definition: jsonb.h:161
void * arg
JsonbValue * findJsonbValueFromContainer(JsonbContainer *container, uint32 flags, JsonbValue *key)
Definition: jsonb_util.c:327
uint32 nElems
Definition: jsonb.h:328
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:328
#define elog
Definition: elog.h:219
static void convertJsonbObject(StringInfo buffer, JEntry *header, JsonbValue *val, int level)
Definition: jsonb_util.c:1554
static unsigned hash(unsigned *uv, int n)
Definition: rege_dfa.c:541
static int lengthCompareJsonbStringValue(const void *a, const void *b)
Definition: jsonb_util.c:1719
long val
Definition: informix.c:689
#define DirectFunctionCall2(func, arg1, arg2)
Definition: fmgr.h:586
JsonbIteratorToken JsonbIteratorNext(JsonbIterator **it, JsonbValue *val, bool skipNested)
Definition: jsonb_util.c:754
Definition: jsonb.h:25
static void appendKey(JsonbParseState *pstate, JsonbValue *scalarVal)
Definition: jsonb_util.c:647