PostgreSQL Source Code  git master
jsonb_plpython.c
Go to the documentation of this file.
1 #include "postgres.h"
2 
3 #include "plpython.h"
4 #include "plpy_elog.h"
5 #include "plpy_typeio.h"
6 #include "utils/jsonb.h"
7 #include "utils/fmgrprotos.h"
8 #include "utils/numeric.h"
9 
11 
12 void _PG_init(void);
13 
14 /* for PLyObject_AsString in plpy_typeio.c */
15 typedef char *(*PLyObject_AsString_t) (PyObject *plrv);
17 
18 typedef void (*PLy_elog_impl_t) (int elevel, const char *fmt,...);
20 
21 /*
22  * decimal_constructor is a function from python library and used
23  * for transforming strings into python decimal type
24  */
25 static PyObject *decimal_constructor;
26 
27 static PyObject *PLyObject_FromJsonbContainer(JsonbContainer *jsonb);
28 static JsonbValue *PLyObject_ToJsonbValue(PyObject *obj,
29  JsonbParseState **jsonb_state, bool is_elem);
30 
31 #if PY_MAJOR_VERSION >= 3
32 typedef PyObject *(*PLyUnicode_FromStringAndSize_t)
33  (const char *s, Py_ssize_t size);
34 static PLyUnicode_FromStringAndSize_t PLyUnicode_FromStringAndSize_p;
35 #endif
36 
37 /*
38  * Module initialize function: fetch function pointers for cross-module calls.
39  */
40 void
41 _PG_init(void)
42 {
43  /* Asserts verify that typedefs above match original declarations */
46  load_external_function("$libdir/" PLPYTHON_LIBNAME, "PLyObject_AsString",
47  true, NULL);
48 #if PY_MAJOR_VERSION >= 3
49  AssertVariableIsOfType(&PLyUnicode_FromStringAndSize, PLyUnicode_FromStringAndSize_t);
50  PLyUnicode_FromStringAndSize_p = (PLyUnicode_FromStringAndSize_t)
51  load_external_function("$libdir/" PLPYTHON_LIBNAME, "PLyUnicode_FromStringAndSize",
52  true, NULL);
53 #endif
54 
57  load_external_function("$libdir/" PLPYTHON_LIBNAME, "PLy_elog_impl",
58  true, NULL);
59 }
60 
61 /* These defines must be after the _PG_init */
62 #define PLyObject_AsString (PLyObject_AsString_p)
63 #define PLyUnicode_FromStringAndSize (PLyUnicode_FromStringAndSize_p)
64 #undef PLy_elog
65 #define PLy_elog (PLy_elog_impl_p)
66 
67 /*
68  * PLyString_FromJsonbValue
69  *
70  * Transform string JsonbValue to Python string.
71  */
72 static PyObject *
74 {
75  Assert(jbv->type == jbvString);
76 
77  return PyString_FromStringAndSize(jbv->val.string.val, jbv->val.string.len);
78 }
79 
80 /*
81  * PLyString_ToJsonbValue
82  *
83  * Transform Python string to JsonbValue.
84  */
85 static void
86 PLyString_ToJsonbValue(PyObject *obj, JsonbValue *jbvElem)
87 {
88  jbvElem->type = jbvString;
89  jbvElem->val.string.val = PLyObject_AsString(obj);
90  jbvElem->val.string.len = strlen(jbvElem->val.string.val);
91 }
92 
93 /*
94  * PLyObject_FromJsonbValue
95  *
96  * Transform JsonbValue to PyObject.
97  */
98 static PyObject *
100 {
101  switch (jsonbValue->type)
102  {
103  case jbvNull:
104  Py_RETURN_NONE;
105 
106  case jbvBinary:
107  return PLyObject_FromJsonbContainer(jsonbValue->val.binary.data);
108 
109  case jbvNumeric:
110  {
111  Datum num;
112  char *str;
113 
114  num = NumericGetDatum(jsonbValue->val.numeric);
116 
117  return PyObject_CallFunction(decimal_constructor, "s", str);
118  }
119 
120  case jbvString:
121  return PLyString_FromJsonbValue(jsonbValue);
122 
123  case jbvBool:
124  if (jsonbValue->val.boolean)
125  Py_RETURN_TRUE;
126  else
127  Py_RETURN_FALSE;
128 
129  default:
130  elog(ERROR, "unexpected jsonb value type: %d", jsonbValue->type);
131  return NULL;
132  }
133 }
134 
135 /*
136  * PLyObject_FromJsonbContainer
137  *
138  * Transform JsonbContainer to PyObject.
139  */
140 static PyObject *
142 {
144  JsonbValue v;
145  JsonbIterator *it;
146  PyObject *result;
147 
148  it = JsonbIteratorInit(jsonb);
149  r = JsonbIteratorNext(&it, &v, true);
150 
151  switch (r)
152  {
153  case WJB_BEGIN_ARRAY:
154  if (v.val.array.rawScalar)
155  {
156  JsonbValue tmp;
157 
158  if ((r = JsonbIteratorNext(&it, &v, true)) != WJB_ELEM ||
159  (r = JsonbIteratorNext(&it, &tmp, true)) != WJB_END_ARRAY ||
160  (r = JsonbIteratorNext(&it, &tmp, true)) != WJB_DONE)
161  elog(ERROR, "unexpected jsonb token: %d", r);
162 
163  result = PLyObject_FromJsonbValue(&v);
164  }
165  else
166  {
167  PyObject *volatile elem = NULL;
168 
169  result = PyList_New(0);
170  if (!result)
171  return NULL;
172 
173  PG_TRY();
174  {
175  while ((r = JsonbIteratorNext(&it, &v, true)) != WJB_DONE)
176  {
177  if (r != WJB_ELEM)
178  continue;
179 
180  elem = PLyObject_FromJsonbValue(&v);
181 
182  PyList_Append(result, elem);
183  Py_XDECREF(elem);
184  elem = NULL;
185  }
186  }
187  PG_CATCH();
188  {
189  Py_XDECREF(elem);
190  Py_XDECREF(result);
191  PG_RE_THROW();
192  }
193  PG_END_TRY();
194  }
195  break;
196 
197  case WJB_BEGIN_OBJECT:
198  {
199  PyObject *volatile result_v = PyDict_New();
200  PyObject *volatile key = NULL;
201  PyObject *volatile val = NULL;
202 
203  if (!result_v)
204  return NULL;
205 
206  PG_TRY();
207  {
208  while ((r = JsonbIteratorNext(&it, &v, true)) != WJB_DONE)
209  {
210  if (r != WJB_KEY)
211  continue;
212 
213  key = PLyString_FromJsonbValue(&v);
214  if (!key)
215  {
216  Py_XDECREF(result_v);
217  result_v = NULL;
218  break;
219  }
220 
221  if ((r = JsonbIteratorNext(&it, &v, true)) != WJB_VALUE)
222  elog(ERROR, "unexpected jsonb token: %d", r);
223 
224  val = PLyObject_FromJsonbValue(&v);
225  if (!val)
226  {
227  Py_XDECREF(key);
228  key = NULL;
229  Py_XDECREF(result_v);
230  result_v = NULL;
231  break;
232  }
233 
234  PyDict_SetItem(result_v, key, val);
235 
236  Py_XDECREF(key);
237  key = NULL;
238  Py_XDECREF(val);
239  val = NULL;
240  }
241  }
242  PG_CATCH();
243  {
244  Py_XDECREF(result_v);
245  Py_XDECREF(key);
246  Py_XDECREF(val);
247  PG_RE_THROW();
248  }
249  PG_END_TRY();
250 
251  result = result_v;
252  }
253  break;
254 
255  default:
256  elog(ERROR, "unexpected jsonb token: %d", r);
257  return NULL;
258  }
259 
260  return result;
261 }
262 
263 /*
264  * PLyMapping_ToJsonbValue
265  *
266  * Transform Python dict to JsonbValue.
267  */
268 static JsonbValue *
269 PLyMapping_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state)
270 {
271  Py_ssize_t pcount;
272  PyObject *volatile items;
273  JsonbValue *volatile out;
274 
275  pcount = PyMapping_Size(obj);
276  items = PyMapping_Items(obj);
277 
278  PG_TRY();
279  {
280  Py_ssize_t i;
281 
282  pushJsonbValue(jsonb_state, WJB_BEGIN_OBJECT, NULL);
283 
284  for (i = 0; i < pcount; i++)
285  {
286  JsonbValue jbvKey;
287  PyObject *item = PyList_GetItem(items, i);
288  PyObject *key = PyTuple_GetItem(item, 0);
289  PyObject *value = PyTuple_GetItem(item, 1);
290 
291  /* Python dictionary can have None as key */
292  if (key == Py_None)
293  {
294  jbvKey.type = jbvString;
295  jbvKey.val.string.len = 0;
296  jbvKey.val.string.val = "";
297  }
298  else
299  {
300  /* All others types of keys we serialize to string */
301  PLyString_ToJsonbValue(key, &jbvKey);
302  }
303 
304  (void) pushJsonbValue(jsonb_state, WJB_KEY, &jbvKey);
305  (void) PLyObject_ToJsonbValue(value, jsonb_state, false);
306  }
307 
308  out = pushJsonbValue(jsonb_state, WJB_END_OBJECT, NULL);
309  }
310  PG_CATCH();
311  {
312  Py_DECREF(items);
313  PG_RE_THROW();
314  }
315  PG_END_TRY();
316 
317  Py_DECREF(items);
318 
319  return out;
320 }
321 
322 /*
323  * PLySequence_ToJsonbValue
324  *
325  * Transform python list to JsonbValue. Expects transformed PyObject and
326  * a state required for jsonb construction.
327  */
328 static JsonbValue *
329 PLySequence_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state)
330 {
331  Py_ssize_t i;
332  Py_ssize_t pcount;
333  PyObject *volatile value = NULL;
334 
335  pcount = PySequence_Size(obj);
336 
337  pushJsonbValue(jsonb_state, WJB_BEGIN_ARRAY, NULL);
338 
339  PG_TRY();
340  {
341  for (i = 0; i < pcount; i++)
342  {
343  value = PySequence_GetItem(obj, i);
344  Assert(value);
345 
346  (void) PLyObject_ToJsonbValue(value, jsonb_state, true);
347  Py_XDECREF(value);
348  value = NULL;
349  }
350  }
351  PG_CATCH();
352  {
353  Py_XDECREF(value);
354  PG_RE_THROW();
355  }
356  PG_END_TRY();
357 
358  return pushJsonbValue(jsonb_state, WJB_END_ARRAY, NULL);
359 }
360 
361 /*
362  * PLyNumber_ToJsonbValue(PyObject *obj)
363  *
364  * Transform python number to JsonbValue.
365  */
366 static JsonbValue *
367 PLyNumber_ToJsonbValue(PyObject *obj, JsonbValue *jbvNum)
368 {
369  Numeric num;
370  char *str = PLyObject_AsString(obj);
371 
372  PG_TRY();
373  {
374  Datum numd;
375 
377  CStringGetDatum(str),
379  Int32GetDatum(-1));
380  num = DatumGetNumeric(numd);
381  }
382  PG_CATCH();
383  {
384  ereport(ERROR,
385  (errcode(ERRCODE_DATATYPE_MISMATCH),
386  (errmsg("could not convert value \"%s\" to jsonb", str))));
387  }
388  PG_END_TRY();
389 
390  pfree(str);
391 
392  /*
393  * jsonb doesn't allow NaN (per JSON specification), so we have to prevent
394  * it here explicitly. (Infinity is also not allowed in jsonb, but
395  * numeric_in above already catches that.)
396  */
397  if (numeric_is_nan(num))
398  ereport(ERROR,
399  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
400  (errmsg("cannot convert NaN to jsonb"))));
401 
402  jbvNum->type = jbvNumeric;
403  jbvNum->val.numeric = num;
404 
405  return jbvNum;
406 }
407 
408 /*
409  * PLyObject_ToJsonbValue(PyObject *obj)
410  *
411  * Transform python object to JsonbValue.
412  */
413 static JsonbValue *
414 PLyObject_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state, bool is_elem)
415 {
416  JsonbValue buf;
417  JsonbValue *out;
418 
419  if (!(PyString_Check(obj) || PyUnicode_Check(obj)))
420  {
421  if (PySequence_Check(obj))
422  return PLySequence_ToJsonbValue(obj, jsonb_state);
423  else if (PyMapping_Check(obj))
424  return PLyMapping_ToJsonbValue(obj, jsonb_state);
425  }
426 
427  /* Allocate JsonbValue in heap only if it is raw scalar value. */
428  if (*jsonb_state)
429  out = &buf;
430  else
431  out = palloc(sizeof(JsonbValue));
432 
433  if (obj == Py_None)
434  out->type = jbvNull;
435  else if (PyString_Check(obj) || PyUnicode_Check(obj))
436  PLyString_ToJsonbValue(obj, out);
437 
438  /*
439  * PyNumber_Check() returns true for booleans, so boolean check should
440  * come first.
441  */
442  else if (PyBool_Check(obj))
443  {
444  out->type = jbvBool;
445  out->val.boolean = (obj == Py_True);
446  }
447  else if (PyNumber_Check(obj))
448  out = PLyNumber_ToJsonbValue(obj, out);
449  else
450  ereport(ERROR,
451  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
452  (errmsg("Python type \"%s\" cannot be transformed to jsonb",
453  PLyObject_AsString((PyObject *) obj->ob_type)))));
454 
455  /* Push result into 'jsonb_state' unless it is raw scalar value. */
456  return (*jsonb_state ?
457  pushJsonbValue(jsonb_state, is_elem ? WJB_ELEM : WJB_VALUE, out) :
458  out);
459 }
460 
461 /*
462  * plpython_to_jsonb
463  *
464  * Transform python object to Jsonb datum
465  */
467 Datum
469 {
470  PyObject *obj;
471  JsonbValue *out;
472  JsonbParseState *jsonb_state = NULL;
473 
474  obj = (PyObject *) PG_GETARG_POINTER(0);
475  out = PLyObject_ToJsonbValue(obj, &jsonb_state, true);
477 }
478 
479 /*
480  * jsonb_to_plpython
481  *
482  * Transform Jsonb datum to PyObject and return it as internal.
483  */
485 Datum
487 {
488  PyObject *result;
489  Jsonb *in = PG_GETARG_JSONB_P(0);
490 
491  /*
492  * Initialize pointer to Decimal constructor. First we try "cdecimal", C
493  * version of decimal library. In case of failure we use slower "decimal"
494  * module.
495  */
496  if (!decimal_constructor)
497  {
498  PyObject *decimal_module = PyImport_ImportModule("cdecimal");
499 
500  if (!decimal_module)
501  {
502  PyErr_Clear();
503  decimal_module = PyImport_ImportModule("decimal");
504  }
505  Assert(decimal_module);
506  decimal_constructor = PyObject_GetAttrString(decimal_module, "Decimal");
507  }
508 
509  result = PLyObject_FromJsonbContainer(&in->root);
510  if (!result)
511  PLy_elog(ERROR, "transformation from jsonb to Python failed");
512 
513  return PointerGetDatum(result);
514 }
#define PG_RETURN_POINTER(x)
Definition: fmgr.h:351
static JsonbValue * PLyMapping_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state)
Jsonb * JsonbValueToJsonb(JsonbValue *val)
Definition: jsonb_util.c:81
Definition: jsonb.h:220
#define PointerGetDatum(X)
Definition: postgres.h:556
char * val
Definition: jsonb.h:264
#define NumericGetDatum(X)
Definition: numeric.h:51
Definition: jsonb.h:239
static struct @144 value
Datum numeric_out(PG_FUNCTION_ARGS)
Definition: numeric.c:655
Definition: jsonb.h:22
void PLy_elog_impl(int elevel, const char *fmt,...)
Definition: plpy_elog.c:47
int errcode(int sqlerrcode)
Definition: elog.c:570
#define PG_GETARG_POINTER(n)
Definition: fmgr.h:271
Definition: jsonb.h:236
#define DirectFunctionCall1(func, arg1)
Definition: fmgr.h:617
PG_FUNCTION_INFO_V1(plpython_to_jsonb)
int Py_ssize_t
Definition: plpython.h:67
JsonbValue * pushJsonbValue(JsonbParseState **pstate, JsonbIteratorToken seq, JsonbValue *jbval)
Definition: jsonb_util.c:551
void pfree(void *pointer)
Definition: mcxt.c:1031
void _PG_init(void)
static void PLyString_ToJsonbValue(PyObject *obj, JsonbValue *jbvElem)
#define ObjectIdGetDatum(X)
Definition: postgres.h:507
#define ERROR
Definition: elog.h:43
static PyObject * decimal_constructor
bool numeric_is_nan(Numeric num)
Definition: numeric.c:683
#define DatumGetCString(X)
Definition: postgres.h:566
#define PLy_elog
static char * buf
Definition: pg_test_fsync.c:68
#define PLyObject_AsString
#define CStringGetDatum(X)
Definition: postgres.h:578
void(* PLy_elog_impl_t)(int elevel, const char *fmt,...)
Datum numeric_in(PG_FUNCTION_ARGS)
Definition: numeric.c:573
Definition: jsonb.h:23
PGFunction load_external_function(const char *filename, const char *funcname, bool signalNotFound, void **filehandle)
Definition: dfmgr.c:107
#define ereport(elevel, rest)
Definition: elog.h:141
JsonbIteratorToken
Definition: jsonb.h:20
#define DirectFunctionCall3(func, arg1, arg2, arg3)
Definition: fmgr.h:621
JsonbContainer root
Definition: jsonb.h:223
static int elevel
Definition: vacuumlazy.c:143
uintptr_t Datum
Definition: postgres.h:367
static PyObject * PLyObject_FromJsonbContainer(JsonbContainer *jsonb)
static JsonbValue * PLyNumber_ToJsonbValue(PyObject *obj, JsonbValue *jbvNum)
#define InvalidOid
Definition: postgres_ext.h:36
PG_MODULE_MAGIC
static PyObject * PLyString_FromJsonbValue(JsonbValue *jbv)
char *(* PLyObject_AsString_t)(PyObject *plrv)
#define PG_CATCH()
Definition: elog.h:310
#define DatumGetNumeric(X)
Definition: numeric.h:49
Datum plpython_to_jsonb(PG_FUNCTION_ARGS)
#define Assert(condition)
Definition: c.h:732
JsonbIterator * JsonbIteratorInit(JsonbContainer *container)
Definition: jsonb_util.c:752
#define PLyUnicode_FromStringAndSize
static PLy_elog_impl_t PLy_elog_impl_p
static JsonbValue * PLyObject_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state, bool is_elem)
#define PG_RE_THROW()
Definition: elog.h:331
enum jbvType type
Definition: jsonb.h:255
Datum jsonb_to_plpython(PG_FUNCTION_ARGS)
static PLyObject_AsString_t PLyObject_AsString_p
#define Int32GetDatum(X)
Definition: postgres.h:479
void * palloc(Size size)
Definition: mcxt.c:924
int errmsg(const char *fmt,...)
Definition: elog.c:784
#define elog(elevel,...)
Definition: elog.h:226
int i
#define PG_FUNCTION_ARGS
Definition: fmgr.h:188
#define PG_TRY()
Definition: elog.h:301
static JsonbValue * PLySequence_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state)
long val
Definition: informix.c:684
#define PG_END_TRY()
Definition: elog.h:317
JsonbIteratorToken JsonbIteratorNext(JsonbIterator **it, JsonbValue *val, bool skipNested)
Definition: jsonb_util.c:788
Definition: jsonb.h:25
#define PG_GETARG_JSONB_P(x)
Definition: jsonb.h:74
static PyObject * PLyObject_FromJsonbValue(JsonbValue *jsonbValue)
#define AssertVariableIsOfType(varname, typename)
Definition: c.h:881