PostgreSQL Source Code  git master
llvmjit.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * llvmjit.c
4  * Core part of the LLVM JIT provider.
5  *
6  * Copyright (c) 2016-2021, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/jit/llvm/llvmjit.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres.h"
15 
16 #include <llvm-c/Analysis.h>
17 #include <llvm-c/BitReader.h>
18 #include <llvm-c/BitWriter.h>
19 #include <llvm-c/Core.h>
20 #include <llvm-c/ExecutionEngine.h>
21 #if LLVM_VERSION_MAJOR > 11
22 #include <llvm-c/Orc.h>
23 #include <llvm-c/OrcEE.h>
24 #include <llvm-c/LLJIT.h>
25 #else
26 #include <llvm-c/OrcBindings.h>
27 #endif
28 #include <llvm-c/Support.h>
29 #include <llvm-c/Target.h>
30 #include <llvm-c/Transforms/IPO.h>
31 #include <llvm-c/Transforms/PassManagerBuilder.h>
32 #include <llvm-c/Transforms/Scalar.h>
33 #if LLVM_VERSION_MAJOR > 6
34 #include <llvm-c/Transforms/Utils.h>
35 #endif
36 
37 #include "jit/llvmjit.h"
38 #include "jit/llvmjit_emit.h"
39 #include "miscadmin.h"
40 #include "portability/instr_time.h"
41 #include "storage/ipc.h"
42 #include "utils/memutils.h"
43 #include "utils/resowner_private.h"
44 
45 /* Handle of a module emitted via ORC JIT */
46 typedef struct LLVMJitHandle
47 {
48 #if LLVM_VERSION_MAJOR > 11
49  LLVMOrcLLJITRef lljit;
50  LLVMOrcResourceTrackerRef resource_tracker;
51 #else
52  LLVMOrcJITStackRef stack;
53  LLVMOrcModuleHandle orc_handle;
54 #endif
56 
57 
58 /* types & functions commonly needed for JITing */
59 LLVMTypeRef TypeSizeT;
60 LLVMTypeRef TypeParamBool;
61 LLVMTypeRef TypeStorageBool;
62 LLVMTypeRef TypePGFunction;
63 LLVMTypeRef StructNullableDatum;
68 LLVMTypeRef StructHeapTupleData;
71 LLVMTypeRef StructBlockId;
73 LLVMTypeRef StructTupleConstr;
74 LLVMTypeRef StructTupleDescData;
79 LLVMTypeRef StructPGFinfoRecord;
80 LLVMTypeRef StructFmgrInfo;
82 LLVMTypeRef StructExprContext;
83 LLVMTypeRef StructExprEvalStep;
84 LLVMTypeRef StructExprState;
85 LLVMTypeRef StructAggState;
88 
89 LLVMValueRef AttributeTemplate;
90 
91 LLVMModuleRef llvm_types_module = NULL;
92 
93 static bool llvm_session_initialized = false;
94 static size_t llvm_generation = 0;
95 static const char *llvm_triple = NULL;
96 static const char *llvm_layout = NULL;
97 
98 
99 static LLVMTargetRef llvm_targetref;
100 #if LLVM_VERSION_MAJOR > 11
101 static LLVMOrcThreadSafeContextRef llvm_ts_context;
102 static LLVMOrcLLJITRef llvm_opt0_orc;
103 static LLVMOrcLLJITRef llvm_opt3_orc;
104 #else /* LLVM_VERSION_MAJOR > 11 */
105 static LLVMOrcJITStackRef llvm_opt0_orc;
106 static LLVMOrcJITStackRef llvm_opt3_orc;
107 #endif /* LLVM_VERSION_MAJOR > 11 */
108 
109 
110 static void llvm_release_context(JitContext *context);
111 static void llvm_session_initialize(void);
112 static void llvm_shutdown(int code, Datum arg);
113 static void llvm_compile_module(LLVMJitContext *context);
114 static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
115 
116 static void llvm_create_types(void);
117 static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
118 
119 #if LLVM_VERSION_MAJOR > 11
120 static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
121 static char *llvm_error_message(LLVMErrorRef error);
122 #endif /* LLVM_VERSION_MAJOR > 11 */
123 
125 
126 
127 /*
128  * Initialize LLVM JIT provider.
129  */
130 void
132 {
136 }
137 
138 /*
139  * Create a context for JITing work.
140  *
141  * The context, including subsidiary resources, will be cleaned up either when
142  * the context is explicitly released, or when the lifetime of
143  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
144  */
145 LLVMJitContext *
146 llvm_create_context(int jitFlags)
147 {
148  LLVMJitContext *context;
149 
151 
153 
155 
157  sizeof(LLVMJitContext));
158  context->base.flags = jitFlags;
159 
160  /* ensure cleanup */
161  context->base.resowner = CurrentResourceOwner;
163 
164  return context;
165 }
166 
167 /*
168  * Release resources required by one llvm context.
169  */
170 static void
172 {
173  LLVMJitContext *llvm_context = (LLVMJitContext *) context;
174 
175  /*
176  * When this backend is exiting, don't clean up LLVM. As an error might
177  * have occurred from within LLVM, we do not want to risk reentering. All
178  * resource cleanup is going to happen through process exit.
179  */
181  return;
182 
184 
185  if (llvm_context->module)
186  {
187  LLVMDisposeModule(llvm_context->module);
188  llvm_context->module = NULL;
189  }
190 
191  while (llvm_context->handles != NIL)
192  {
193  LLVMJitHandle *jit_handle;
194 
195  jit_handle = (LLVMJitHandle *) linitial(llvm_context->handles);
196  llvm_context->handles = list_delete_first(llvm_context->handles);
197 
198 #if LLVM_VERSION_MAJOR > 11
199  {
200  LLVMOrcExecutionSessionRef ee;
201  LLVMOrcSymbolStringPoolRef sp;
202 
203  LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
204  LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
205 
206  /*
207  * Without triggering cleanup of the string pool, we'd leak
208  * memory. It'd be sufficient to do this far less often, but in
209  * experiments the required time was small enough to just always
210  * do it.
211  */
212  ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
213  sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
214  LLVMOrcSymbolStringPoolClearDeadEntries(sp);
215  }
216 #else /* LLVM_VERSION_MAJOR > 11 */
217  {
218  LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
219  }
220 #endif /* LLVM_VERSION_MAJOR > 11 */
221 
222  pfree(jit_handle);
223  }
224 }
225 
226 /*
227  * Return module which may be modified, e.g. by creating new functions.
228  */
229 LLVMModuleRef
230 llvm_mutable_module(LLVMJitContext *context)
231 {
233 
234  /*
235  * If there's no in-progress module, create a new one.
236  */
237  if (!context->module)
238  {
239  context->compiled = false;
240  context->module_generation = llvm_generation++;
241  context->module = LLVMModuleCreateWithName("pg");
242  LLVMSetTarget(context->module, llvm_triple);
243  LLVMSetDataLayout(context->module, llvm_layout);
244  }
245 
246  return context->module;
247 }
248 
249 /*
250  * Expand function name to be non-conflicting. This should be used by code
251  * generating code, when adding new externally visible function definitions to
252  * a Module.
253  */
254 char *
255 llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
256 {
257  Assert(context->module != NULL);
258 
259  context->base.instr.created_functions++;
260 
261  /*
262  * Previously we used dots to separate, but turns out some tools, e.g.
263  * GDB, don't like that and truncate name.
264  */
265  return psprintf("%s_%zu_%d",
266  basename,
267  context->module_generation,
268  context->counter++);
269 }
270 
271 /*
272  * Return pointer to function funcname, which has to exist. If there's pending
273  * code to be optimized and emitted, do so first.
274  */
275 void *
276 llvm_get_function(LLVMJitContext *context, const char *funcname)
277 {
278 #if LLVM_VERSION_MAJOR > 11 || \
279  defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
280  ListCell *lc;
281 #endif
282 
284 
285  /*
286  * If there is a pending / not emitted module, compile and emit now.
287  * Otherwise we might not find the [correct] function.
288  */
289  if (!context->compiled)
290  {
291  llvm_compile_module(context);
292  }
293 
294  /*
295  * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
296  * to mangle here.
297  */
298 
299 #if LLVM_VERSION_MAJOR > 11
300  foreach(lc, context->handles)
301  {
302  LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
303  instr_time starttime;
304  instr_time endtime;
305  LLVMErrorRef error;
306  LLVMOrcJITTargetAddress addr;
307 
308  INSTR_TIME_SET_CURRENT(starttime);
309 
310  addr = 0;
311  error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
312  if (error)
313  elog(ERROR, "failed to look up symbol \"%s\": %s",
314  funcname, llvm_error_message(error));
315 
316  /*
317  * LLJIT only actually emits code the first time a symbol is
318  * referenced. Thus add lookup time to emission time. That's counting
319  * a bit more than with older LLVM versions, but unlikely to ever
320  * matter.
321  */
322  INSTR_TIME_SET_CURRENT(endtime);
323  INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
324  endtime, starttime);
325 
326  if (addr)
327  return (void *) (uintptr_t) addr;
328  }
329 #elif defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
330  foreach(lc, context->handles)
331  {
332  LLVMOrcTargetAddress addr;
333  LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
334 
335  addr = 0;
336  if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
337  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
338  if (addr)
339  return (void *) (uintptr_t) addr;
340  }
341 #elif LLVM_VERSION_MAJOR < 5
342  {
343  LLVMOrcTargetAddress addr;
344 
345  if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
346  return (void *) (uintptr_t) addr;
347  if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
348  return (void *) (uintptr_t) addr;
349  }
350 #else
351  {
352  LLVMOrcTargetAddress addr;
353 
354  if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
355  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
356  if (addr)
357  return (void *) (uintptr_t) addr;
358  if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
359  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
360  if (addr)
361  return (void *) (uintptr_t) addr;
362  }
363 #endif
364 
365  elog(ERROR, "failed to JIT: %s", funcname);
366 
367  return NULL;
368 }
369 
370 /*
371  * Return type of a variable in llvmjit_types.c. This is useful to keep types
372  * in sync between plain C and JIT related code.
373  */
374 LLVMTypeRef
375 llvm_pg_var_type(const char *varname)
376 {
377  LLVMValueRef v_srcvar;
378  LLVMTypeRef typ;
379 
380  /* this'll return a *pointer* to the global */
381  v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
382  if (!v_srcvar)
383  elog(ERROR, "variable %s not in llvmjit_types.c", varname);
384 
385  /* look at the contained type */
386  typ = LLVMTypeOf(v_srcvar);
387  Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMPointerTypeKind);
388  typ = LLVMGetElementType(typ);
389  Assert(typ != NULL);
390 
391  return typ;
392 }
393 
394 /*
395  * Return function type of a variable in llvmjit_types.c. This is useful to
396  * keep function types in sync between C and JITed code.
397  */
398 LLVMTypeRef
399 llvm_pg_var_func_type(const char *varname)
400 {
401  LLVMTypeRef typ = llvm_pg_var_type(varname);
402 
403  /* look at the contained type */
404  Assert(LLVMGetTypeKind(typ) == LLVMPointerTypeKind);
405  typ = LLVMGetElementType(typ);
406  Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMFunctionTypeKind);
407 
408  return typ;
409 }
410 
411 /*
412  * Return declaration for a function referenced in llvmjit_types.c, adding it
413  * to the module if necessary.
414  *
415  * This is used to make functions discovered via llvm_create_types() known to
416  * the module that's currently being worked on.
417  */
418 LLVMValueRef
419 llvm_pg_func(LLVMModuleRef mod, const char *funcname)
420 {
421  LLVMValueRef v_srcfn;
422  LLVMValueRef v_fn;
423 
424  /* don't repeatedly add function */
425  v_fn = LLVMGetNamedFunction(mod, funcname);
426  if (v_fn)
427  return v_fn;
428 
429  v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
430 
431  if (!v_srcfn)
432  elog(ERROR, "function %s not in llvmjit_types.c", funcname);
433 
434  v_fn = LLVMAddFunction(mod,
435  funcname,
436  LLVMGetElementType(LLVMTypeOf(v_srcfn)));
437  llvm_copy_attributes(v_srcfn, v_fn);
438 
439  return v_fn;
440 }
441 
442 /*
443  * Copy attributes from one function to another, for a specific index (an
444  * index can reference return value, function and parameter attributes).
445  */
446 static void
447 llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
448 {
449  int num_attributes;
450  LLVMAttributeRef *attrs;
451 
452  num_attributes = LLVMGetAttributeCountAtIndexPG(v_from, index);
453 
454  /*
455  * Not just for efficiency: LLVM <= 3.9 crashes when
456  * LLVMGetAttributesAtIndex() is called for an index with 0 attributes.
457  */
458  if (num_attributes == 0)
459  return;
460 
461  attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
462  LLVMGetAttributesAtIndex(v_from, index, attrs);
463 
464  for (int attno = 0; attno < num_attributes; attno++)
465  LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
466 
467  pfree(attrs);
468 }
469 
470 /*
471  * Copy all attributes from one function to another. I.e. function, return and
472  * parameters will be copied.
473  */
474 void
475 llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
476 {
477  uint32 param_count;
478 
479  /* copy function attributes */
480  llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
481 
482  /* and the return value attributes */
483  llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
484 
485  /* and each function parameter's attribute */
486  param_count = LLVMCountParams(v_from);
487 
488  for (int paramidx = 1; paramidx <= param_count; paramidx++)
489  llvm_copy_attributes_at_index(v_from, v_to, paramidx);
490 }
491 
492 /*
493  * Return a callable LLVMValueRef for fcinfo.
494  */
495 LLVMValueRef
496 llvm_function_reference(LLVMJitContext *context,
497  LLVMBuilderRef builder,
498  LLVMModuleRef mod,
499  FunctionCallInfo fcinfo)
500 {
501  char *modname;
502  char *basename;
503  char *funcname;
504 
505  LLVMValueRef v_fn;
506 
507  fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
508 
509  if (modname != NULL && basename != NULL)
510  {
511  /* external function in loadable library */
512  funcname = psprintf("pgextern.%s.%s", modname, basename);
513  }
514  else if (basename != NULL)
515  {
516  /* internal function */
517  funcname = psprintf("%s", basename);
518  }
519  else
520  {
521  /*
522  * Function we don't know to handle, return pointer. We do so by
523  * creating a global constant containing a pointer to the function.
524  * Makes IR more readable.
525  */
526  LLVMValueRef v_fn_addr;
527 
528  funcname = psprintf("pgoidextern.%u",
529  fcinfo->flinfo->fn_oid);
530  v_fn = LLVMGetNamedGlobal(mod, funcname);
531  if (v_fn != 0)
532  return LLVMBuildLoad(builder, v_fn, "");
533 
534  v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
535 
536  v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
537  LLVMSetInitializer(v_fn, v_fn_addr);
538  LLVMSetGlobalConstant(v_fn, true);
539  LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
540  LLVMSetUnnamedAddr(v_fn, true);
541 
542  return LLVMBuildLoad(builder, v_fn, "");
543  }
544 
545  /* check if function already has been added */
546  v_fn = LLVMGetNamedFunction(mod, funcname);
547  if (v_fn != 0)
548  return v_fn;
549 
550  v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction));
551 
552  return v_fn;
553 }
554 
555 /*
556  * Optimize code in module using the flags set in context.
557  */
558 static void
559 llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
560 {
561  LLVMPassManagerBuilderRef llvm_pmb;
562  LLVMPassManagerRef llvm_mpm;
563  LLVMPassManagerRef llvm_fpm;
564  LLVMValueRef func;
565  int compile_optlevel;
566 
567  if (context->base.flags & PGJIT_OPT3)
568  compile_optlevel = 3;
569  else
570  compile_optlevel = 0;
571 
572  /*
573  * Have to create a new pass manager builder every pass through, as the
574  * inliner has some per-builder state. Otherwise one ends up only inlining
575  * a function the first time though.
576  */
577  llvm_pmb = LLVMPassManagerBuilderCreate();
578  LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
579  llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
580 
581  if (context->base.flags & PGJIT_OPT3)
582  {
583  /* TODO: Unscientifically determined threshold */
584  LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
585  }
586  else
587  {
588  /* we rely on mem2reg heavily, so emit even in the O0 case */
589  LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
590  }
591 
592  LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
593 
594  /*
595  * Do function level optimization. This could be moved to the point where
596  * functions are emitted, to reduce memory usage a bit.
597  */
598  LLVMInitializeFunctionPassManager(llvm_fpm);
599  for (func = LLVMGetFirstFunction(context->module);
600  func != NULL;
601  func = LLVMGetNextFunction(func))
602  LLVMRunFunctionPassManager(llvm_fpm, func);
603  LLVMFinalizeFunctionPassManager(llvm_fpm);
604  LLVMDisposePassManager(llvm_fpm);
605 
606  /*
607  * Perform module level optimization. We do so even in the non-optimized
608  * case, so always-inline functions etc get inlined. It's cheap enough.
609  */
610  llvm_mpm = LLVMCreatePassManager();
611  LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
612  llvm_mpm);
613  /* always use always-inliner pass */
614  if (!(context->base.flags & PGJIT_OPT3))
615  LLVMAddAlwaysInlinerPass(llvm_mpm);
616  /* if doing inlining, but no expensive optimization, add inlining pass */
617  if (context->base.flags & PGJIT_INLINE
618  && !(context->base.flags & PGJIT_OPT3))
619  LLVMAddFunctionInliningPass(llvm_mpm);
620  LLVMRunPassManager(llvm_mpm, context->module);
621  LLVMDisposePassManager(llvm_mpm);
622 
623  LLVMPassManagerBuilderDispose(llvm_pmb);
624 }
625 
626 /*
627  * Emit code for the currently pending module.
628  */
629 static void
630 llvm_compile_module(LLVMJitContext *context)
631 {
632  LLVMJitHandle *handle;
633  MemoryContext oldcontext;
634  instr_time starttime;
635  instr_time endtime;
636 #if LLVM_VERSION_MAJOR > 11
637  LLVMOrcLLJITRef compile_orc;
638 #else
639  LLVMOrcJITStackRef compile_orc;
640 #endif
641 
642  if (context->base.flags & PGJIT_OPT3)
643  compile_orc = llvm_opt3_orc;
644  else
645  compile_orc = llvm_opt0_orc;
646 
647  /* perform inlining */
648  if (context->base.flags & PGJIT_INLINE)
649  {
650  INSTR_TIME_SET_CURRENT(starttime);
651  llvm_inline(context->module);
652  INSTR_TIME_SET_CURRENT(endtime);
653  INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
654  endtime, starttime);
655  }
656 
657  if (jit_dump_bitcode)
658  {
659  char *filename;
660 
661  filename = psprintf("%u.%zu.bc",
662  MyProcPid,
663  context->module_generation);
664  LLVMWriteBitcodeToFile(context->module, filename);
665  pfree(filename);
666  }
667 
668 
669  /* optimize according to the chosen optimization settings */
670  INSTR_TIME_SET_CURRENT(starttime);
671  llvm_optimize_module(context, context->module);
672  INSTR_TIME_SET_CURRENT(endtime);
673  INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
674  endtime, starttime);
675 
676  if (jit_dump_bitcode)
677  {
678  char *filename;
679 
680  filename = psprintf("%u.%zu.optimized.bc",
681  MyProcPid,
682  context->module_generation);
683  LLVMWriteBitcodeToFile(context->module, filename);
684  pfree(filename);
685  }
686 
687  handle = (LLVMJitHandle *)
689 
690  /*
691  * Emit the code. Note that this can, depending on the optimization
692  * settings, take noticeable resources as code emission executes low-level
693  * instruction combining/selection passes etc. Without optimization a
694  * faster instruction selection mechanism is used.
695  */
696  INSTR_TIME_SET_CURRENT(starttime);
697 #if LLVM_VERSION_MAJOR > 11
698  {
699  LLVMOrcThreadSafeModuleRef ts_module;
700  LLVMErrorRef error;
701  LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
702 
703  ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
704 
705  handle->lljit = compile_orc;
706  handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
707 
708  /*
709  * NB: This doesn't actually emit code. That happens lazily the first
710  * time a symbol defined in the module is requested. Due to that
711  * llvm_get_function() also accounts for emission time.
712  */
713 
714  context->module = NULL; /* will be owned by LLJIT */
715  error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
716  handle->resource_tracker,
717  ts_module);
718 
719  if (error)
720  elog(ERROR, "failed to JIT module: %s",
721  llvm_error_message(error));
722 
723  handle->lljit = compile_orc;
724 
725  /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
726  }
727 #elif LLVM_VERSION_MAJOR > 6
728  {
729  handle->stack = compile_orc;
730  if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
731  llvm_resolve_symbol, NULL))
732  elog(ERROR, "failed to JIT module");
733 
734  /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
735  }
736 #elif LLVM_VERSION_MAJOR > 4
737  {
738  LLVMSharedModuleRef smod;
739 
740  smod = LLVMOrcMakeSharedModule(context->module);
741  handle->stack = compile_orc;
742  if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, smod,
743  llvm_resolve_symbol, NULL))
744  elog(ERROR, "failed to JIT module");
745 
746  LLVMOrcDisposeSharedModuleRef(smod);
747  }
748 #else /* LLVM 4.0 and 3.9 */
749  {
750  handle->stack = compile_orc;
751  handle->orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
752  llvm_resolve_symbol, NULL);
753 
754  LLVMDisposeModule(context->module);
755  }
756 #endif
757 
758  INSTR_TIME_SET_CURRENT(endtime);
759  INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
760  endtime, starttime);
761 
762  context->module = NULL;
763  context->compiled = true;
764 
765  /* remember emitted code for cleanup and lookups */
767  context->handles = lappend(context->handles, handle);
768  MemoryContextSwitchTo(oldcontext);
769 
770  ereport(DEBUG1,
771  (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
772  INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
773  INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
774  INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
775  errhidestmt(true),
776  errhidecontext(true)));
777 }
778 
779 /*
780  * Per session initialization.
781  */
782 static void
784 {
785  MemoryContext oldcontext;
786  char *error = NULL;
787  char *cpu = NULL;
788  char *features = NULL;
789  LLVMTargetMachineRef opt0_tm;
790  LLVMTargetMachineRef opt3_tm;
791 
793  return;
794 
796 
797  LLVMInitializeNativeTarget();
798  LLVMInitializeNativeAsmPrinter();
799  LLVMInitializeNativeAsmParser();
800 
801  /*
802  * Synchronize types early, as that also includes inferring the target
803  * triple.
804  */
806 
807  if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
808  {
809  elog(FATAL, "failed to query triple %s\n", error);
810  }
811 
812  /*
813  * We want the generated code to use all available features. Therefore
814  * grab the host CPU string and detect features of the current CPU. The
815  * latter is needed because some CPU architectures default to enabling
816  * features not all CPUs have (weird, huh).
817  */
818  cpu = LLVMGetHostCPUName();
819  features = LLVMGetHostCPUFeatures();
820  elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
821  cpu, features);
822 
823  opt0_tm =
824  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
825  LLVMCodeGenLevelNone,
826  LLVMRelocDefault,
827  LLVMCodeModelJITDefault);
828  opt3_tm =
829  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
830  LLVMCodeGenLevelAggressive,
831  LLVMRelocDefault,
832  LLVMCodeModelJITDefault);
833 
834  LLVMDisposeMessage(cpu);
835  cpu = NULL;
836  LLVMDisposeMessage(features);
837  features = NULL;
838 
839  /* force symbols in main binary to be loaded */
840  LLVMLoadLibraryPermanently(NULL);
841 
842 #if LLVM_VERSION_MAJOR > 11
843  {
844  llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
845 
846  llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
847  opt0_tm = 0;
848 
849  llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
850  opt3_tm = 0;
851  }
852 #else /* LLVM_VERSION_MAJOR > 11 */
853  {
854  llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
855  llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
856 
857 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
859  {
860  LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
861 
862  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
863  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
864  }
865 #endif
866 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
868  {
869  LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
870 
871  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
872  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
873  }
874 #endif
875  }
876 #endif /* LLVM_VERSION_MAJOR > 11 */
877 
879 
881 
882  MemoryContextSwitchTo(oldcontext);
883 }
884 
885 static void
887 {
888  /*
889  * If llvm_shutdown() is reached while in a fatal-on-oom section an error
890  * has occurred in the middle of LLVM code. It is not safe to call back
891  * into LLVM (which is why a FATAL error was thrown).
892  *
893  * We do need to shutdown LLVM in other shutdown cases, otherwise
894  * e.g. profiling data won't be written out.
895  */
896  if (llvm_in_fatal_on_oom())
897  {
899  return;
900  }
901 
902 #if LLVM_VERSION_MAJOR > 11
903  {
904  if (llvm_opt3_orc)
905  {
906  LLVMOrcDisposeLLJIT(llvm_opt3_orc);
907  llvm_opt3_orc = NULL;
908  }
909  if (llvm_opt0_orc)
910  {
911  LLVMOrcDisposeLLJIT(llvm_opt0_orc);
912  llvm_opt0_orc = NULL;
913  }
914  if (llvm_ts_context)
915  {
916  LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
917  llvm_ts_context = NULL;
918  }
919  }
920 #else /* LLVM_VERSION_MAJOR > 11 */
921  {
922  /* unregister profiling support, needs to be flushed to be useful */
923 
924  if (llvm_opt3_orc)
925  {
926 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
928  LLVMOrcUnregisterPerf(llvm_opt3_orc);
929 #endif
930  LLVMOrcDisposeInstance(llvm_opt3_orc);
931  llvm_opt3_orc = NULL;
932  }
933 
934  if (llvm_opt0_orc)
935  {
936 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
938  LLVMOrcUnregisterPerf(llvm_opt0_orc);
939 #endif
940  LLVMOrcDisposeInstance(llvm_opt0_orc);
941  llvm_opt0_orc = NULL;
942  }
943  }
944 #endif /* LLVM_VERSION_MAJOR > 11 */
945 }
946 
947 /* helper for llvm_create_types, returning a function's return type */
948 static LLVMTypeRef
949 load_return_type(LLVMModuleRef mod, const char *name)
950 {
951  LLVMValueRef value;
952  LLVMTypeRef typ;
953 
954  /* this'll return a *pointer* to the function */
955  value = LLVMGetNamedFunction(mod, name);
956  if (!value)
957  elog(ERROR, "function %s is unknown", name);
958 
959  /* get type of function pointer */
960  typ = LLVMTypeOf(value);
961  Assert(typ != NULL);
962  /* dereference pointer */
963  typ = LLVMGetElementType(typ);
964  Assert(typ != NULL);
965  /* and look at return type */
966  typ = LLVMGetReturnType(typ);
967  Assert(typ != NULL);
968 
969  return typ;
970 }
971 
972 /*
973  * Load required information, types, function signatures from llvmjit_types.c
974  * and make them available in global variables.
975  *
976  * Those global variables are then used while emitting code.
977  */
978 static void
980 {
981  char path[MAXPGPATH];
982  LLVMMemoryBufferRef buf;
983  char *msg;
984 
985  snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
986 
987  /* open file */
988  if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
989  {
990  elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
991  path, msg);
992  }
993 
994  /* eagerly load contents, going to need it all */
995  if (LLVMParseBitcode2(buf, &llvm_types_module))
996  {
997  elog(ERROR, "LLVMParseBitcode2 of %s failed", path);
998  }
999  LLVMDisposeMemoryBuffer(buf);
1000 
1001  /*
1002  * Load triple & layout from clang emitted file so we're guaranteed to be
1003  * compatible.
1004  */
1005  llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
1006  llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
1007 
1008  TypeSizeT = llvm_pg_var_type("TypeSizeT");
1009  TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
1010  TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
1011  TypePGFunction = llvm_pg_var_type("TypePGFunction");
1012  StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
1013  StructExprContext = llvm_pg_var_type("StructExprContext");
1014  StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
1015  StructExprState = llvm_pg_var_type("StructExprState");
1016  StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
1017  StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
1018  StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
1019  StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
1020  StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
1021  StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
1022  StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
1023  StructAggState = llvm_pg_var_type("StructAggState");
1024  StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
1025  StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
1026 
1027  AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
1028 }
1029 
1030 /*
1031  * Split a symbol into module / function parts. If the function is in the
1032  * main binary (or an external library) *modname will be NULL.
1033  */
1034 void
1035 llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1036 {
1037  *modname = NULL;
1038  *funcname = NULL;
1039 
1040  /*
1041  * Module function names are pgextern.$module.$funcname
1042  */
1043  if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1044  {
1045  /*
1046  * Symbol names cannot contain a ., therefore we can split based on
1047  * first and last occurrence of one.
1048  */
1049  *funcname = rindex(name, '.');
1050  (*funcname)++; /* jump over . */
1051 
1052  *modname = pnstrdup(name + strlen("pgextern."),
1053  *funcname - name - strlen("pgextern.") - 1);
1054  Assert(funcname);
1055 
1056  *funcname = pstrdup(*funcname);
1057  }
1058  else
1059  {
1060  *modname = NULL;
1061  *funcname = pstrdup(name);
1062  }
1063 }
1064 
1065 /*
1066  * Attempt to resolve symbol, so LLVM can emit a reference to it.
1067  */
1068 static uint64_t
1069 llvm_resolve_symbol(const char *symname, void *ctx)
1070 {
1071  uintptr_t addr;
1072  char *funcname;
1073  char *modname;
1074 
1075  /*
1076  * macOS prefixes all object level symbols with an underscore. But neither
1077  * dlsym() nor PG's inliner expect that. So undo.
1078  */
1079 #if defined(__darwin__)
1080  if (symname[0] != '_')
1081  elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1082  symname++;
1083 #endif
1084 
1085  llvm_split_symbol_name(symname, &modname, &funcname);
1086 
1087  /* functions that aren't resolved to names shouldn't ever get here */
1088  Assert(funcname);
1089 
1090  if (modname)
1091  addr = (uintptr_t) load_external_function(modname, funcname,
1092  true, NULL);
1093  else
1094  addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1095 
1096  pfree(funcname);
1097  if (modname)
1098  pfree(modname);
1099 
1100  /* let LLVM will error out - should never happen */
1101  if (!addr)
1102  elog(WARNING, "failed to resolve name %s", symname);
1103 
1104  return (uint64_t) addr;
1105 }
1106 
1107 #if LLVM_VERSION_MAJOR > 11
1108 
1109 static LLVMErrorRef
1110 llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1111  LLVMOrcLookupStateRef * LookupState, LLVMOrcLookupKind Kind,
1112  LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1113  LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1114 {
1115  LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1116  LLVMErrorRef error;
1117  LLVMOrcMaterializationUnitRef mu;
1118 
1119  for (int i = 0; i < LookupSetSize; i++)
1120  {
1121  const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1122 
1123 #if LLVM_VERSION_MAJOR > 12
1124  LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1125 #endif
1126  symbols[i].Name = LookupSet[i].Name;
1127  symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1128  symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1129  }
1130 
1131  mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1132  error = LLVMOrcJITDylibDefine(JD, mu);
1133  if (error != LLVMErrorSuccess)
1134  LLVMOrcDisposeMaterializationUnit(mu);
1135 
1136  pfree(symbols);
1137 
1138  return error;
1139 }
1140 
1141 /*
1142  * We cannot throw errors through LLVM (without causing a FATAL at least), so
1143  * just use WARNING here. That's OK anyway, as the error is also reported at
1144  * the top level action (with less detail) and there might be multiple
1145  * invocations of errors with details.
1146  *
1147  * This doesn't really happen during normal operation, but in cases like
1148  * symbol resolution breakage. So just using elog(WARNING) is fine.
1149  */
1150 static void
1151 llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1152 {
1153  elog(WARNING, "error during JITing: %s",
1154  llvm_error_message(error));
1155 }
1156 
1157 /*
1158  * Create our own object layer, so we can add event listeners.
1159  */
1160 static LLVMOrcObjectLayerRef
1161 llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1162 {
1163  LLVMOrcObjectLayerRef objlayer =
1164  LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1165 
1166 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1168  {
1169  LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1170 
1171  LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1172  }
1173 #endif
1174 
1175 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1177  {
1178  LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1179 
1180  LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1181  }
1182 #endif
1183 
1184  return objlayer;
1185 }
1186 
1187 /*
1188  * Create LLJIT instance, using the passed in target machine. Note that the
1189  * target machine afterwards is owned by the LLJIT instance.
1190  */
1191 static LLVMOrcLLJITRef
1192 llvm_create_jit_instance(LLVMTargetMachineRef tm)
1193 {
1194  LLVMOrcLLJITRef lljit;
1195  LLVMOrcJITTargetMachineBuilderRef tm_builder;
1196  LLVMOrcLLJITBuilderRef lljit_builder;
1197  LLVMErrorRef error;
1198  LLVMOrcDefinitionGeneratorRef main_gen;
1199  LLVMOrcDefinitionGeneratorRef ref_gen;
1200 
1201  lljit_builder = LLVMOrcCreateLLJITBuilder();
1202  tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1203  LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1204 
1205  LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1206  llvm_create_object_layer,
1207  NULL);
1208 
1209  error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1210  if (error)
1211  elog(ERROR, "failed to create lljit instance: %s",
1212  llvm_error_message(error));
1213 
1214  LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1215  llvm_log_jit_error, NULL);
1216 
1217  /*
1218  * Symbol resolution support for symbols in the postgres binary /
1219  * libraries already loaded.
1220  */
1221  error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1222  LLVMOrcLLJITGetGlobalPrefix(lljit),
1223  0, NULL);
1224  if (error)
1225  elog(ERROR, "failed to create generator: %s",
1226  llvm_error_message(error));
1227  LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1228 
1229  /*
1230  * Symbol resolution support for "special" functions, e.g. a call into an
1231  * SQL callable function.
1232  */
1233  ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1234  LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1235 
1236  return lljit;
1237 }
1238 
1239 static char *
1240 llvm_error_message(LLVMErrorRef error)
1241 {
1242  char *orig = LLVMGetErrorMessage(error);
1243  char *msg = pstrdup(orig);
1244 
1245  LLVMDisposeErrorMessage(orig);
1246 
1247  return msg;
1248 }
1249 
1250 #endif /* LLVM_VERSION_MAJOR > 11 */
LLVMTypeRef StructPGFinfoRecord
Definition: llvmjit.c:79
#define NIL
Definition: pg_list.h:65
JitProviderResetAfterErrorCB reset_after_error
Definition: jit.h:75
LLVMTypeRef StructTupleConstr
Definition: llvmjit.c:73
#define DEBUG1
Definition: elog.h:25
LLVMTypeRef StructFunctionCallInfoData
Definition: llvmjit.c:81
int MyProcPid
Definition: globals.c:43
static void llvm_compile_module(LLVMJitContext *context)
Definition: llvmjit.c:630
char * pnstrdup(const char *in, Size len)
Definition: mcxt.c:1310
PG_MODULE_MAGIC
Definition: llvmjit.c:124
static void error(void)
Definition: sql-dyntest.c:147
LLVMTypeRef StructHeapTupleFieldsField3
Definition: llvmjit.c:64
LLVMTypeRef StructMinimalTupleData
Definition: llvmjit.c:69
LLVMOrcJITStackRef stack
Definition: llvmjit.c:52
PGFunction fn_addr
Definition: fmgr.h:58
void on_proc_exit(pg_on_exit_callback function, Datum arg)
Definition: ipc.c:305
LLVMTypeRef StructExprEvalStep
Definition: llvmjit.c:83
#define PointerGetDatum(X)
Definition: postgres.h:600
LLVMTypeRef TypeSizeT
Definition: llvmjit.c:59
LLVMTypeRef StructBlockId
Definition: llvmjit.c:71
LLVMTypeRef StructExprState
Definition: llvmjit.c:84
LLVMTypeRef StructHeapTupleFields
Definition: llvmjit.c:65
ResourceOwner CurrentResourceOwner
Definition: resowner.c:146
char * pstrdup(const char *in)
Definition: mcxt.c:1299
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
bool jit_debugging_support
Definition: jit.c:34
struct timeval instr_time
Definition: instr_time.h:150
static void llvm_shutdown(int code, Datum arg)
Definition: llvmjit.c:886
LLVMTypeRef StructNullableDatum
Definition: llvmjit.c:63
LLVMTypeRef StructItemPointerData
Definition: llvmjit.c:70
Definition: jit.h:54
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
void _PG_jit_provider_init(JitProviderCallbacks *cb)
Definition: llvmjit.c:131
LLVMValueRef llvm_function_reference(LLVMJitContext *context, LLVMBuilderRef builder, LLVMModuleRef mod, FunctionCallInfo fcinfo)
Definition: llvmjit.c:496
int errhidestmt(bool hide_stmt)
Definition: elog.c:1250
#define INSTR_TIME_ACCUM_DIFF(x, y, z)
Definition: instr_time.h:182
static const char * llvm_layout
Definition: llvmjit.c:96
LLVMJitContext * llvm_create_context(int jitFlags)
Definition: llvmjit.c:146
LLVMTypeRef StructFmgrInfo
Definition: llvmjit.c:80
#define INSTR_TIME_GET_DOUBLE(t)
Definition: instr_time.h:199
LLVMTypeRef StructHeapTupleTableSlot
Definition: llvmjit.c:76
LLVMTypeRef StructAggState
Definition: llvmjit.c:85
static LLVMTypeRef load_return_type(LLVMModuleRef mod, const char *name)
Definition: llvmjit.c:949
void fmgr_symbol(Oid functionId, char **mod, char **fn)
Definition: fmgr.c:283
static struct pg_tm tm
Definition: localtime.c:102
static const char * llvm_triple
Definition: llvmjit.c:95
Definition: type.h:89
LLVMTypeRef llvm_pg_var_type(const char *varname)
Definition: llvmjit.c:375
LLVMTypeRef StructHeapTupleData
Definition: llvmjit.c:68
LLVMTypeRef StructHeapTupleDataChoice
Definition: llvmjit.c:67
bool llvm_in_fatal_on_oom(void)
#define PGJIT_OPT3
Definition: jit.h:21
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
Definition: llvmjit.c:1035
void pfree(void *pointer)
Definition: mcxt.c:1169
#define linitial(l)
Definition: pg_list.h:174
#define ERROR
Definition: elog.h:46
struct LLVMJitHandle LLVMJitHandle
#define FATAL
Definition: elog.h:49
static uint64_t llvm_resolve_symbol(const char *name, void *ctx)
Definition: llvmjit.c:1069
#define MAXPGPATH
unsigned LLVMGetAttributeCountAtIndexPG(LLVMValueRef F, uint32 Idx)
LLVMOrcModuleHandle orc_handle
Definition: llvmjit.c:53
char * llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
Definition: llvmjit.c:255
Definition: c.h:675
#define DEBUG2
Definition: elog.h:24
void llvm_assert_in_fatal_section(void)
static char * buf
Definition: pg_test_fsync.c:68
LLVMTypeRef TypeStorageBool
Definition: llvmjit.c:61
unsigned int uint32
Definition: c.h:441
LLVMTypeRef StructMemoryContextData
Definition: llvmjit.c:78
LLVMTypeRef StructExprContext
Definition: llvmjit.c:82
MemoryContext TopMemoryContext
Definition: mcxt.c:48
static LLVMOrcJITStackRef llvm_opt3_orc
Definition: llvmjit.c:106
void * load_external_function(const char *filename, const char *funcname, bool signalNotFound, void **filehandle)
Definition: dfmgr.c:107
List * lappend(List *list, void *datum)
Definition: list.c:336
LLVMTypeRef TypePGFunction
Definition: llvmjit.c:62
#define WARNING
Definition: elog.h:40
LLVMTypeRef StructAggStatePerTransData
Definition: llvmjit.c:87
JitProviderCompileExprCB compile_expr
Definition: jit.h:77
void ResourceOwnerRememberJIT(ResourceOwner owner, Datum handle)
Definition: resowner.c:1387
void * palloc0(Size size)
Definition: mcxt.c:1093
uintptr_t Datum
Definition: postgres.h:411
FmgrInfo * flinfo
Definition: fmgr.h:87
#define PGJIT_INLINE
Definition: jit.h:22
static bool llvm_session_initialized
Definition: llvmjit.c:93
void llvm_reset_after_error(void)
LLVMTypeRef StructMinimalTupleTableSlot
Definition: llvmjit.c:77
static size_t llvm_generation
Definition: llvmjit.c:94
void * MemoryContextAllocZero(MemoryContext context, Size size)
Definition: mcxt.c:906
LLVMTypeRef StructTupleTableSlot
Definition: llvmjit.c:75
static struct @143 value
Oid fn_oid
Definition: fmgr.h:59
#define ereport(elevel,...)
Definition: elog.h:157
LLVMModuleRef llvm_mutable_module(LLVMJitContext *context)
Definition: llvmjit.c:230
static LLVMTargetRef llvm_targetref
Definition: llvmjit.c:99
int errmsg_internal(const char *fmt,...)
Definition: elog.c:996
static void llvm_session_initialize(void)
Definition: llvmjit.c:783
static void llvm_release_context(JitContext *context)
Definition: llvmjit.c:171
#define Assert(condition)
Definition: c.h:804
#define lfirst(lc)
Definition: pg_list.h:169
bool proc_exit_inprogress
Definition: ipc.c:40
bool llvm_compile_expr(ExprState *state)
Definition: llvmjit_expr.c:78
LLVMTypeRef StructTupleDescData
Definition: llvmjit.c:74
LLVMTypeRef TypeParamBool
Definition: llvmjit.c:60
bool jit_profiling_support
Definition: jit.c:37
void * llvm_get_function(LLVMJitContext *context, const char *funcname)
Definition: llvmjit.c:276
bool jit_dump_bitcode
Definition: jit.c:35
#define INSTR_TIME_SET_CURRENT(t)
Definition: instr_time.h:156
const char * name
Definition: encode.c:561
static char * filename
Definition: pg_dumpall.c:92
void llvm_inline(LLVMModuleRef M)
static void llvm_create_types(void)
Definition: llvmjit.c:979
void * palloc(Size size)
Definition: mcxt.c:1062
void * MemoryContextAlloc(MemoryContext context, Size size)
Definition: mcxt.c:863
LLVMModuleRef llvm_types_module
Definition: llvmjit.c:91
void llvm_enter_fatal_on_oom(void)
#define elog(elevel,...)
Definition: elog.h:232
int i
LLVMValueRef AttributeTemplate
Definition: llvmjit.c:89
void * arg
static void llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
Definition: llvmjit.c:447
LLVMTypeRef StructHeapTupleHeaderData
Definition: llvmjit.c:66
void ResourceOwnerEnlargeJIT(ResourceOwner owner)
Definition: resowner.c:1376
void llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
Definition: llvmjit.c:475
static LLVMOrcJITStackRef llvm_opt0_orc
Definition: llvmjit.c:105
JitProviderReleaseContextCB release_context
Definition: jit.h:76
#define snprintf
Definition: port.h:216
static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
Definition: llvmjit.c:559
char pkglib_path[MAXPGPATH]
Definition: globals.c:76
int errhidecontext(bool hide_ctx)
Definition: elog.c:1269
List * list_delete_first(List *list)
Definition: list.c:875
LLVMTypeRef llvm_pg_var_func_type(const char *varname)
Definition: llvmjit.c:399
LLVMTypeRef StructFormPgAttribute
Definition: llvmjit.c:72
LLVMValueRef llvm_pg_func(LLVMModuleRef mod, const char *funcname)
Definition: llvmjit.c:419
LLVMTypeRef StructAggStatePerGroupData
Definition: llvmjit.c:86