PostgreSQL Source Code  git master
llvmjit.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * llvmjit.c
4  * Core part of the LLVM JIT provider.
5  *
6  * Copyright (c) 2016-2024, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/jit/llvm/llvmjit.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres.h"
15 
16 #include <llvm-c/Analysis.h>
17 #include <llvm-c/BitReader.h>
18 #include <llvm-c/BitWriter.h>
19 #include <llvm-c/Core.h>
20 #include <llvm-c/ExecutionEngine.h>
21 #if LLVM_VERSION_MAJOR > 16
22 #include <llvm-c/Transforms/PassBuilder.h>
23 #endif
24 #if LLVM_VERSION_MAJOR > 11
25 #include <llvm-c/Orc.h>
26 #include <llvm-c/OrcEE.h>
27 #include <llvm-c/LLJIT.h>
28 #else
29 #include <llvm-c/OrcBindings.h>
30 #endif
31 #include <llvm-c/Support.h>
32 #include <llvm-c/Target.h>
33 #if LLVM_VERSION_MAJOR < 17
34 #include <llvm-c/Transforms/IPO.h>
35 #include <llvm-c/Transforms/PassManagerBuilder.h>
36 #include <llvm-c/Transforms/Scalar.h>
37 #include <llvm-c/Transforms/Utils.h>
38 #endif
39 
40 #include "jit/llvmjit.h"
41 #include "jit/llvmjit_emit.h"
42 #include "miscadmin.h"
43 #include "portability/instr_time.h"
44 #include "storage/ipc.h"
45 #include "utils/memutils.h"
46 #include "utils/resowner.h"
47 
48 #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
49 
50 /* Handle of a module emitted via ORC JIT */
51 typedef struct LLVMJitHandle
52 {
53 #if LLVM_VERSION_MAJOR > 11
54  LLVMOrcLLJITRef lljit;
55  LLVMOrcResourceTrackerRef resource_tracker;
56 #else
57  LLVMOrcJITStackRef stack;
58  LLVMOrcModuleHandle orc_handle;
59 #endif
61 
62 
63 /* types & functions commonly needed for JITing */
64 LLVMTypeRef TypeSizeT;
65 LLVMTypeRef TypeParamBool;
66 LLVMTypeRef TypeStorageBool;
67 LLVMTypeRef TypePGFunction;
68 LLVMTypeRef StructNullableDatum;
69 LLVMTypeRef StructHeapTupleData;
71 LLVMTypeRef StructTupleDescData;
78 LLVMTypeRef StructExprContext;
79 LLVMTypeRef StructExprEvalStep;
80 LLVMTypeRef StructExprState;
81 LLVMTypeRef StructAggState;
84 LLVMTypeRef StructPlanState;
85 
86 LLVMValueRef AttributeTemplate;
89 
90 static LLVMModuleRef llvm_types_module = NULL;
91 
92 static bool llvm_session_initialized = false;
93 static size_t llvm_generation = 0;
94 
95 /* number of LLVMJitContexts that currently are in use */
97 
98 /* how many times has the current LLVMContextRef been used */
100 static const char *llvm_triple = NULL;
101 static const char *llvm_layout = NULL;
102 static LLVMContextRef llvm_context;
103 
104 
105 static LLVMTargetRef llvm_targetref;
106 #if LLVM_VERSION_MAJOR > 11
107 static LLVMOrcThreadSafeContextRef llvm_ts_context;
108 static LLVMOrcLLJITRef llvm_opt0_orc;
109 static LLVMOrcLLJITRef llvm_opt3_orc;
110 #else /* LLVM_VERSION_MAJOR > 11 */
111 static LLVMOrcJITStackRef llvm_opt0_orc;
112 static LLVMOrcJITStackRef llvm_opt3_orc;
113 #endif /* LLVM_VERSION_MAJOR > 11 */
114 
115 
116 static void llvm_release_context(JitContext *context);
117 static void llvm_session_initialize(void);
118 static void llvm_shutdown(int code, Datum arg);
119 static void llvm_compile_module(LLVMJitContext *context);
120 static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
121 
122 static void llvm_create_types(void);
123 static void llvm_set_target(void);
124 static void llvm_recreate_llvm_context(void);
125 static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
126 
127 #if LLVM_VERSION_MAJOR > 11
128 static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
129 static char *llvm_error_message(LLVMErrorRef error);
130 #endif /* LLVM_VERSION_MAJOR > 11 */
131 
132 /* ResourceOwner callbacks to hold JitContexts */
133 static void ResOwnerReleaseJitContext(Datum res);
134 
136 {
137  .name = "LLVM JIT context",
138  .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
139  .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
140  .ReleaseResource = ResOwnerReleaseJitContext,
141  .DebugPrint = NULL /* the default message is fine */
142 };
143 
144 /* Convenience wrappers over ResourceOwnerRemember/Forget */
145 static inline void
146 ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
147 {
149 }
150 static inline void
151 ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
152 {
154 }
155 
157 
158 
159 /*
160  * Initialize LLVM JIT provider.
161  */
162 void
164 {
168 }
169 
170 
171 /*
172  * Every now and then create a new LLVMContextRef. Unfortunately, during every
173  * round of inlining, types may "leak" (they can still be found/used via the
174  * context, but new types will be created the next time in inlining is
175  * performed). To prevent that from slowly accumulating problematic amounts of
176  * memory, recreate the LLVMContextRef we use. We don't want to do so too
177  * often, as that implies some overhead (particularly re-loading the module
178  * summaries / modules is fairly expensive). A future TODO would be to make
179  * this more finegrained and only drop/recreate the LLVMContextRef when we know
180  * there has been inlining. If we can get the size of the context from LLVM
181  * then that might be a better way to determine when to drop/recreate rather
182  * then the usagecount heuristic currently employed.
183  */
184 static void
186 {
187  if (!llvm_context)
188  elog(ERROR, "Trying to recreate a non-existing context");
189 
190  /*
191  * We can only safely recreate the LLVM context if no other code is being
192  * JITed, otherwise we'd release the types in use for that.
193  */
195  {
197  return;
198  }
199 
201  {
203  return;
204  }
205 
206  /*
207  * Need to reset the modules that the inlining code caches before
208  * disposing of the context. LLVM modules exist within a specific LLVM
209  * context, therefore disposing of the context before resetting the cache
210  * would lead to dangling pointers to modules.
211  */
213 
214  LLVMContextDispose(llvm_context);
215  llvm_context = LLVMContextCreate();
217 
218  /*
219  * Re-build cached type information, so code generation code can rely on
220  * that information to be present (also prevents the variables to be
221  * dangling references).
222  */
224 }
225 
226 
227 /*
228  * Create a context for JITing work.
229  *
230  * The context, including subsidiary resources, will be cleaned up either when
231  * the context is explicitly released, or when the lifetime of
232  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
233  */
234 LLVMJitContext *
235 llvm_create_context(int jitFlags)
236 {
237  LLVMJitContext *context;
238 
240 
242 
244 
246 
248  sizeof(LLVMJitContext));
249  context->base.flags = jitFlags;
250 
251  /* ensure cleanup */
252  context->base.resowner = CurrentResourceOwner;
254 
256 
257  return context;
258 }
259 
260 /*
261  * Release resources required by one llvm context.
262  */
263 static void
265 {
266  LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
267  ListCell *lc;
268 
269  /*
270  * Consider as cleaned up even if we skip doing so below, that way we can
271  * verify the tracking is correct (see llvm_shutdown()).
272  */
274 
275  /*
276  * When this backend is exiting, don't clean up LLVM. As an error might
277  * have occurred from within LLVM, we do not want to risk reentering. All
278  * resource cleanup is going to happen through process exit.
279  */
281  return;
282 
284 
285  if (llvm_jit_context->module)
286  {
287  LLVMDisposeModule(llvm_jit_context->module);
288  llvm_jit_context->module = NULL;
289  }
290 
291  foreach(lc, llvm_jit_context->handles)
292  {
293  LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
294 
295 #if LLVM_VERSION_MAJOR > 11
296  {
297  LLVMOrcExecutionSessionRef ee;
298  LLVMOrcSymbolStringPoolRef sp;
299 
300  LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
301  LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
302 
303  /*
304  * Without triggering cleanup of the string pool, we'd leak
305  * memory. It'd be sufficient to do this far less often, but in
306  * experiments the required time was small enough to just always
307  * do it.
308  */
309  ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
310  sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
311  LLVMOrcSymbolStringPoolClearDeadEntries(sp);
312  }
313 #else /* LLVM_VERSION_MAJOR > 11 */
314  {
315  LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
316  }
317 #endif /* LLVM_VERSION_MAJOR > 11 */
318 
319  pfree(jit_handle);
320  }
321  list_free(llvm_jit_context->handles);
322  llvm_jit_context->handles = NIL;
323 
325 
326  if (context->resowner)
327  ResourceOwnerForgetJIT(context->resowner, llvm_jit_context);
328 }
329 
330 /*
331  * Return module which may be modified, e.g. by creating new functions.
332  */
333 LLVMModuleRef
334 llvm_mutable_module(LLVMJitContext *context)
335 {
337 
338  /*
339  * If there's no in-progress module, create a new one.
340  */
341  if (!context->module)
342  {
343  context->compiled = false;
344  context->module_generation = llvm_generation++;
345  context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
346  LLVMSetTarget(context->module, llvm_triple);
347  LLVMSetDataLayout(context->module, llvm_layout);
348  }
349 
350  return context->module;
351 }
352 
353 /*
354  * Expand function name to be non-conflicting. This should be used by code
355  * generating code, when adding new externally visible function definitions to
356  * a Module.
357  */
358 char *
359 llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
360 {
361  Assert(context->module != NULL);
362 
363  context->base.instr.created_functions++;
364 
365  /*
366  * Previously we used dots to separate, but turns out some tools, e.g.
367  * GDB, don't like that and truncate name.
368  */
369  return psprintf("%s_%zu_%d",
370  basename,
371  context->module_generation,
372  context->counter++);
373 }
374 
375 /*
376  * Return pointer to function funcname, which has to exist. If there's pending
377  * code to be optimized and emitted, do so first.
378  */
379 void *
380 llvm_get_function(LLVMJitContext *context, const char *funcname)
381 {
382  ListCell *lc;
383 
385 
386  /*
387  * If there is a pending / not emitted module, compile and emit now.
388  * Otherwise we might not find the [correct] function.
389  */
390  if (!context->compiled)
391  {
392  llvm_compile_module(context);
393  }
394 
395  /*
396  * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
397  * to mangle here.
398  */
399 
400 #if LLVM_VERSION_MAJOR > 11
401  foreach(lc, context->handles)
402  {
403  LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
404  instr_time starttime;
405  instr_time endtime;
406  LLVMErrorRef error;
407  LLVMOrcJITTargetAddress addr;
408 
409  INSTR_TIME_SET_CURRENT(starttime);
410 
411  addr = 0;
412  error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
413  if (error)
414  elog(ERROR, "failed to look up symbol \"%s\": %s",
415  funcname, llvm_error_message(error));
416 
417  /*
418  * LLJIT only actually emits code the first time a symbol is
419  * referenced. Thus add lookup time to emission time. That's counting
420  * a bit more than with older LLVM versions, but unlikely to ever
421  * matter.
422  */
423  INSTR_TIME_SET_CURRENT(endtime);
424  INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
425  endtime, starttime);
426 
427  if (addr)
428  return (void *) (uintptr_t) addr;
429  }
430 #else
431  foreach(lc, context->handles)
432  {
433  LLVMOrcTargetAddress addr;
434  LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
435 
436  addr = 0;
437  if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
438  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
439  if (addr)
440  return (void *) (uintptr_t) addr;
441  }
442 #endif
443 
444  elog(ERROR, "failed to JIT: %s", funcname);
445 
446  return NULL;
447 }
448 
449 /*
450  * Return type of a variable in llvmjit_types.c. This is useful to keep types
451  * in sync between plain C and JIT related code.
452  */
453 LLVMTypeRef
454 llvm_pg_var_type(const char *varname)
455 {
456  LLVMValueRef v_srcvar;
457  LLVMTypeRef typ;
458 
459  /* this'll return a *pointer* to the global */
460  v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
461  if (!v_srcvar)
462  elog(ERROR, "variable %s not in llvmjit_types.c", varname);
463 
464  typ = LLVMGlobalGetValueType(v_srcvar);
465 
466  return typ;
467 }
468 
469 /*
470  * Return function type of a variable in llvmjit_types.c. This is useful to
471  * keep function types in sync between C and JITed code.
472  */
473 LLVMTypeRef
474 llvm_pg_var_func_type(const char *varname)
475 {
476  LLVMValueRef v_srcvar;
477  LLVMTypeRef typ;
478 
479  v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
480  if (!v_srcvar)
481  elog(ERROR, "function %s not in llvmjit_types.c", varname);
482 
483  typ = LLVMGetFunctionType(v_srcvar);
484 
485  return typ;
486 }
487 
488 /*
489  * Return declaration for a function referenced in llvmjit_types.c, adding it
490  * to the module if necessary.
491  *
492  * This is used to make functions discovered via llvm_create_types() known to
493  * the module that's currently being worked on.
494  */
495 LLVMValueRef
496 llvm_pg_func(LLVMModuleRef mod, const char *funcname)
497 {
498  LLVMValueRef v_srcfn;
499  LLVMValueRef v_fn;
500 
501  /* don't repeatedly add function */
502  v_fn = LLVMGetNamedFunction(mod, funcname);
503  if (v_fn)
504  return v_fn;
505 
506  v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
507 
508  if (!v_srcfn)
509  elog(ERROR, "function %s not in llvmjit_types.c", funcname);
510 
511  v_fn = LLVMAddFunction(mod,
512  funcname,
513  LLVMGetFunctionType(v_srcfn));
514  llvm_copy_attributes(v_srcfn, v_fn);
515 
516  return v_fn;
517 }
518 
519 /*
520  * Copy attributes from one function to another, for a specific index (an
521  * index can reference return value, function and parameter attributes).
522  */
523 static void
524 llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
525 {
526  int num_attributes;
527  LLVMAttributeRef *attrs;
528 
529  num_attributes = LLVMGetAttributeCountAtIndex(v_from, index);
530 
531  if (num_attributes == 0)
532  return;
533 
534  attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
535  LLVMGetAttributesAtIndex(v_from, index, attrs);
536 
537  for (int attno = 0; attno < num_attributes; attno++)
538  LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
539 
540  pfree(attrs);
541 }
542 
543 /*
544  * Copy all attributes from one function to another. I.e. function, return and
545  * parameters will be copied.
546  */
547 void
548 llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
549 {
550  uint32 param_count;
551 
552  /* copy function attributes */
553  llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
554 
555  /* and the return value attributes */
556  llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
557 
558  /* and each function parameter's attribute */
559  param_count = LLVMCountParams(v_from);
560 
561  for (int paramidx = 1; paramidx <= param_count; paramidx++)
562  llvm_copy_attributes_at_index(v_from, v_to, paramidx);
563 }
564 
565 /*
566  * Return a callable LLVMValueRef for fcinfo.
567  */
568 LLVMValueRef
569 llvm_function_reference(LLVMJitContext *context,
570  LLVMBuilderRef builder,
571  LLVMModuleRef mod,
572  FunctionCallInfo fcinfo)
573 {
574  char *modname;
575  char *basename;
576  char *funcname;
577 
578  LLVMValueRef v_fn;
579 
580  fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
581 
582  if (modname != NULL && basename != NULL)
583  {
584  /* external function in loadable library */
585  funcname = psprintf("pgextern.%s.%s", modname, basename);
586  }
587  else if (basename != NULL)
588  {
589  /* internal function */
590  funcname = pstrdup(basename);
591  }
592  else
593  {
594  /*
595  * Function we don't know to handle, return pointer. We do so by
596  * creating a global constant containing a pointer to the function.
597  * Makes IR more readable.
598  */
599  LLVMValueRef v_fn_addr;
600 
601  funcname = psprintf("pgoidextern.%u",
602  fcinfo->flinfo->fn_oid);
603  v_fn = LLVMGetNamedGlobal(mod, funcname);
604  if (v_fn != 0)
605  return l_load(builder, TypePGFunction, v_fn, "");
606 
607  v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
608 
609  v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
610  LLVMSetInitializer(v_fn, v_fn_addr);
611  LLVMSetGlobalConstant(v_fn, true);
612  LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
613  LLVMSetUnnamedAddr(v_fn, true);
614 
615  return l_load(builder, TypePGFunction, v_fn, "");
616  }
617 
618  /* check if function already has been added */
619  v_fn = LLVMGetNamedFunction(mod, funcname);
620  if (v_fn != 0)
621  return v_fn;
622 
623  v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
624 
625  return v_fn;
626 }
627 
628 /*
629  * Optimize code in module using the flags set in context.
630  */
631 static void
632 llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
633 {
634 #if LLVM_VERSION_MAJOR < 17
635  LLVMPassManagerBuilderRef llvm_pmb;
636  LLVMPassManagerRef llvm_mpm;
637  LLVMPassManagerRef llvm_fpm;
638  LLVMValueRef func;
639  int compile_optlevel;
640 
641  if (context->base.flags & PGJIT_OPT3)
642  compile_optlevel = 3;
643  else
644  compile_optlevel = 0;
645 
646  /*
647  * Have to create a new pass manager builder every pass through, as the
648  * inliner has some per-builder state. Otherwise one ends up only inlining
649  * a function the first time though.
650  */
651  llvm_pmb = LLVMPassManagerBuilderCreate();
652  LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
653  llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
654 
655  if (context->base.flags & PGJIT_OPT3)
656  {
657  /* TODO: Unscientifically determined threshold */
658  LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
659  }
660  else
661  {
662  /* we rely on mem2reg heavily, so emit even in the O0 case */
663  LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
664  }
665 
666  LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
667 
668  /*
669  * Do function level optimization. This could be moved to the point where
670  * functions are emitted, to reduce memory usage a bit.
671  */
672  LLVMInitializeFunctionPassManager(llvm_fpm);
673  for (func = LLVMGetFirstFunction(context->module);
674  func != NULL;
675  func = LLVMGetNextFunction(func))
676  LLVMRunFunctionPassManager(llvm_fpm, func);
677  LLVMFinalizeFunctionPassManager(llvm_fpm);
678  LLVMDisposePassManager(llvm_fpm);
679 
680  /*
681  * Perform module level optimization. We do so even in the non-optimized
682  * case, so always-inline functions etc get inlined. It's cheap enough.
683  */
684  llvm_mpm = LLVMCreatePassManager();
685  LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
686  llvm_mpm);
687  /* always use always-inliner pass */
688  if (!(context->base.flags & PGJIT_OPT3))
689  LLVMAddAlwaysInlinerPass(llvm_mpm);
690  /* if doing inlining, but no expensive optimization, add inlining pass */
691  if (context->base.flags & PGJIT_INLINE
692  && !(context->base.flags & PGJIT_OPT3))
693  LLVMAddFunctionInliningPass(llvm_mpm);
694  LLVMRunPassManager(llvm_mpm, context->module);
695  LLVMDisposePassManager(llvm_mpm);
696 
697  LLVMPassManagerBuilderDispose(llvm_pmb);
698 #else
699  LLVMPassBuilderOptionsRef options;
700  LLVMErrorRef err;
701  const char *passes;
702 
703  if (context->base.flags & PGJIT_OPT3)
704  passes = "default<O3>";
705  else
706  passes = "default<O0>,mem2reg";
707 
708  options = LLVMCreatePassBuilderOptions();
709 
710 #ifdef LLVM_PASS_DEBUG
711  LLVMPassBuilderOptionsSetDebugLogging(options, 1);
712 #endif
713 
714  LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
715 
716  err = LLVMRunPasses(module, passes, NULL, options);
717 
718  if (err)
719  elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
720 
721  LLVMDisposePassBuilderOptions(options);
722 #endif
723 }
724 
725 /*
726  * Emit code for the currently pending module.
727  */
728 static void
729 llvm_compile_module(LLVMJitContext *context)
730 {
731  LLVMJitHandle *handle;
732  MemoryContext oldcontext;
733  instr_time starttime;
734  instr_time endtime;
735 #if LLVM_VERSION_MAJOR > 11
736  LLVMOrcLLJITRef compile_orc;
737 #else
738  LLVMOrcJITStackRef compile_orc;
739 #endif
740 
741  if (context->base.flags & PGJIT_OPT3)
742  compile_orc = llvm_opt3_orc;
743  else
744  compile_orc = llvm_opt0_orc;
745 
746  /* perform inlining */
747  if (context->base.flags & PGJIT_INLINE)
748  {
749  INSTR_TIME_SET_CURRENT(starttime);
750  llvm_inline(context->module);
751  INSTR_TIME_SET_CURRENT(endtime);
752  INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
753  endtime, starttime);
754  }
755 
756  if (jit_dump_bitcode)
757  {
758  char *filename;
759 
760  filename = psprintf("%d.%zu.bc",
761  MyProcPid,
762  context->module_generation);
763  LLVMWriteBitcodeToFile(context->module, filename);
764  pfree(filename);
765  }
766 
767 
768  /* optimize according to the chosen optimization settings */
769  INSTR_TIME_SET_CURRENT(starttime);
770  llvm_optimize_module(context, context->module);
771  INSTR_TIME_SET_CURRENT(endtime);
772  INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
773  endtime, starttime);
774 
775  if (jit_dump_bitcode)
776  {
777  char *filename;
778 
779  filename = psprintf("%d.%zu.optimized.bc",
780  MyProcPid,
781  context->module_generation);
782  LLVMWriteBitcodeToFile(context->module, filename);
783  pfree(filename);
784  }
785 
786  handle = (LLVMJitHandle *)
788 
789  /*
790  * Emit the code. Note that this can, depending on the optimization
791  * settings, take noticeable resources as code emission executes low-level
792  * instruction combining/selection passes etc. Without optimization a
793  * faster instruction selection mechanism is used.
794  */
795  INSTR_TIME_SET_CURRENT(starttime);
796 #if LLVM_VERSION_MAJOR > 11
797  {
798  LLVMOrcThreadSafeModuleRef ts_module;
799  LLVMErrorRef error;
800  LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
801 
802  ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
803 
804  handle->lljit = compile_orc;
805  handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
806 
807  /*
808  * NB: This doesn't actually emit code. That happens lazily the first
809  * time a symbol defined in the module is requested. Due to that
810  * llvm_get_function() also accounts for emission time.
811  */
812 
813  context->module = NULL; /* will be owned by LLJIT */
814  error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
815  handle->resource_tracker,
816  ts_module);
817 
818  if (error)
819  elog(ERROR, "failed to JIT module: %s",
820  llvm_error_message(error));
821 
822  /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
823  }
824 #else
825  {
826  handle->stack = compile_orc;
827  if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
828  llvm_resolve_symbol, NULL))
829  elog(ERROR, "failed to JIT module");
830 
831  /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
832  }
833 #endif
834 
835  INSTR_TIME_SET_CURRENT(endtime);
836  INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
837  endtime, starttime);
838 
839  context->module = NULL;
840  context->compiled = true;
841 
842  /* remember emitted code for cleanup and lookups */
844  context->handles = lappend(context->handles, handle);
845  MemoryContextSwitchTo(oldcontext);
846 
847  ereport(DEBUG1,
848  (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
849  INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
850  INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
851  INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
852  errhidestmt(true),
853  errhidecontext(true)));
854 }
855 
856 /*
857  * Per session initialization.
858  */
859 static void
861 {
862  MemoryContext oldcontext;
863  char *error = NULL;
864  char *cpu = NULL;
865  char *features = NULL;
866  LLVMTargetMachineRef opt0_tm;
867  LLVMTargetMachineRef opt3_tm;
868 
870  return;
871 
873 
874  LLVMInitializeNativeTarget();
875  LLVMInitializeNativeAsmPrinter();
876  LLVMInitializeNativeAsmParser();
877 
878  if (llvm_context == NULL)
879  {
880  llvm_context = LLVMContextCreate();
881 
884  }
885 
886  /*
887  * When targeting LLVM 15, turn off opaque pointers for the context we
888  * build our code in. We don't need to do so for other contexts (e.g.
889  * llvm_ts_context). Once the IR is generated, it carries the necessary
890  * information.
891  *
892  * For 16 and above, opaque pointers must be used, and we have special
893  * code for that.
894  */
895 #if LLVM_VERSION_MAJOR == 15
896  LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
897 #endif
898 
899  /*
900  * Synchronize types early, as that also includes inferring the target
901  * triple.
902  */
904 
905  /*
906  * Extract target information from loaded module.
907  */
908  llvm_set_target();
909 
910  if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
911  {
912  elog(FATAL, "failed to query triple %s", error);
913  }
914 
915  /*
916  * We want the generated code to use all available features. Therefore
917  * grab the host CPU string and detect features of the current CPU. The
918  * latter is needed because some CPU architectures default to enabling
919  * features not all CPUs have (weird, huh).
920  */
921  cpu = LLVMGetHostCPUName();
922  features = LLVMGetHostCPUFeatures();
923  elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
924  cpu, features);
925 
926  opt0_tm =
927  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
928  LLVMCodeGenLevelNone,
929  LLVMRelocDefault,
930  LLVMCodeModelJITDefault);
931  opt3_tm =
932  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
933  LLVMCodeGenLevelAggressive,
934  LLVMRelocDefault,
935  LLVMCodeModelJITDefault);
936 
937  LLVMDisposeMessage(cpu);
938  cpu = NULL;
939  LLVMDisposeMessage(features);
940  features = NULL;
941 
942  /* force symbols in main binary to be loaded */
943  LLVMLoadLibraryPermanently(NULL);
944 
945 #if LLVM_VERSION_MAJOR > 11
946  {
947  llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
948 
949  llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
950  opt0_tm = 0;
951 
952  llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
953  opt3_tm = 0;
954  }
955 #else /* LLVM_VERSION_MAJOR > 11 */
956  {
957  llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
958  llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
959 
960 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
962  {
963  LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
964 
965  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
966  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
967  }
968 #endif
969 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
971  {
972  LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
973 
974  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
975  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
976  }
977 #endif
978  }
979 #endif /* LLVM_VERSION_MAJOR > 11 */
980 
982 
984 
985  MemoryContextSwitchTo(oldcontext);
986 }
987 
988 static void
990 {
991  /*
992  * If llvm_shutdown() is reached while in a fatal-on-oom section an error
993  * has occurred in the middle of LLVM code. It is not safe to call back
994  * into LLVM (which is why a FATAL error was thrown).
995  *
996  * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
997  * profiling data won't be written out.
998  */
999  if (llvm_in_fatal_on_oom())
1000  {
1002  return;
1003  }
1004 
1006  elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
1008 
1009 #if LLVM_VERSION_MAJOR > 11
1010  {
1011  if (llvm_opt3_orc)
1012  {
1013  LLVMOrcDisposeLLJIT(llvm_opt3_orc);
1014  llvm_opt3_orc = NULL;
1015  }
1016  if (llvm_opt0_orc)
1017  {
1018  LLVMOrcDisposeLLJIT(llvm_opt0_orc);
1019  llvm_opt0_orc = NULL;
1020  }
1021  if (llvm_ts_context)
1022  {
1023  LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
1024  llvm_ts_context = NULL;
1025  }
1026  }
1027 #else /* LLVM_VERSION_MAJOR > 11 */
1028  {
1029  /* unregister profiling support, needs to be flushed to be useful */
1030 
1031  if (llvm_opt3_orc)
1032  {
1033 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
1035  LLVMOrcUnregisterPerf(llvm_opt3_orc);
1036 #endif
1037  LLVMOrcDisposeInstance(llvm_opt3_orc);
1038  llvm_opt3_orc = NULL;
1039  }
1040 
1041  if (llvm_opt0_orc)
1042  {
1043 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
1045  LLVMOrcUnregisterPerf(llvm_opt0_orc);
1046 #endif
1047  LLVMOrcDisposeInstance(llvm_opt0_orc);
1048  llvm_opt0_orc = NULL;
1049  }
1050  }
1051 #endif /* LLVM_VERSION_MAJOR > 11 */
1052 }
1053 
1054 /* helper for llvm_create_types, returning a function's return type */
1055 static LLVMTypeRef
1056 load_return_type(LLVMModuleRef mod, const char *name)
1057 {
1058  LLVMValueRef value;
1059  LLVMTypeRef typ;
1060 
1061  /* this'll return a *pointer* to the function */
1062  value = LLVMGetNamedFunction(mod, name);
1063  if (!value)
1064  elog(ERROR, "function %s is unknown", name);
1065 
1066  typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
1067 
1068  return typ;
1069 }
1070 
1071 /*
1072  * Load triple & layout from clang emitted file so we're guaranteed to be
1073  * compatible.
1074  */
1075 static void
1077 {
1078  if (!llvm_types_module)
1079  elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
1080 
1081  if (llvm_triple == NULL)
1082  llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
1083 
1084  if (llvm_layout == NULL)
1085  llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
1086 }
1087 
1088 /*
1089  * Load required information, types, function signatures from llvmjit_types.c
1090  * and make them available in global variables.
1091  *
1092  * Those global variables are then used while emitting code.
1093  */
1094 static void
1096 {
1097  char path[MAXPGPATH];
1098  LLVMMemoryBufferRef buf;
1099  char *msg;
1100 
1101  snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
1102 
1103  /* open file */
1104  if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
1105  {
1106  elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
1107  path, msg);
1108  }
1109 
1110  /* eagerly load contents, going to need it all */
1111  if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
1112  {
1113  elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
1114  }
1115  LLVMDisposeMemoryBuffer(buf);
1116 
1117  TypeSizeT = llvm_pg_var_type("TypeSizeT");
1118  TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
1119  TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
1120  TypePGFunction = llvm_pg_var_type("TypePGFunction");
1121  StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
1122  StructExprContext = llvm_pg_var_type("StructExprContext");
1123  StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
1124  StructExprState = llvm_pg_var_type("StructExprState");
1125  StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
1126  StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
1127  StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
1128  StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
1129  StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
1130  StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
1131  StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
1132  StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
1133  StructAggState = llvm_pg_var_type("StructAggState");
1134  StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
1135  StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
1136  StructPlanState = llvm_pg_var_type("StructPlanState");
1137  StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
1138 
1139  AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
1140  ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
1141  ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
1142 }
1143 
1144 /*
1145  * Split a symbol into module / function parts. If the function is in the
1146  * main binary (or an external library) *modname will be NULL.
1147  */
1148 void
1149 llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1150 {
1151  *modname = NULL;
1152  *funcname = NULL;
1153 
1154  /*
1155  * Module function names are pgextern.$module.$funcname
1156  */
1157  if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1158  {
1159  /*
1160  * Symbol names cannot contain a ., therefore we can split based on
1161  * first and last occurrence of one.
1162  */
1163  *funcname = rindex(name, '.');
1164  (*funcname)++; /* jump over . */
1165 
1166  *modname = pnstrdup(name + strlen("pgextern."),
1167  *funcname - name - strlen("pgextern.") - 1);
1168  Assert(funcname);
1169 
1170  *funcname = pstrdup(*funcname);
1171  }
1172  else
1173  {
1174  *modname = NULL;
1175  *funcname = pstrdup(name);
1176  }
1177 }
1178 
1179 /*
1180  * Attempt to resolve symbol, so LLVM can emit a reference to it.
1181  */
1182 static uint64_t
1183 llvm_resolve_symbol(const char *symname, void *ctx)
1184 {
1185  uintptr_t addr;
1186  char *funcname;
1187  char *modname;
1188 
1189  /*
1190  * macOS prefixes all object level symbols with an underscore. But neither
1191  * dlsym() nor PG's inliner expect that. So undo.
1192  */
1193 #if defined(__darwin__)
1194  if (symname[0] != '_')
1195  elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1196  symname++;
1197 #endif
1198 
1199  llvm_split_symbol_name(symname, &modname, &funcname);
1200 
1201  /* functions that aren't resolved to names shouldn't ever get here */
1202  Assert(funcname);
1203 
1204  if (modname)
1205  addr = (uintptr_t) load_external_function(modname, funcname,
1206  true, NULL);
1207  else
1208  addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1209 
1210  pfree(funcname);
1211  if (modname)
1212  pfree(modname);
1213 
1214  /* let LLVM will error out - should never happen */
1215  if (!addr)
1216  elog(WARNING, "failed to resolve name %s", symname);
1217 
1218  return (uint64_t) addr;
1219 }
1220 
1221 #if LLVM_VERSION_MAJOR > 11
1222 
1223 static LLVMErrorRef
1224 llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1225  LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1226  LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1227  LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1228 {
1229 #if LLVM_VERSION_MAJOR > 14
1230  LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
1231 #else
1232  LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1233 #endif
1234  LLVMErrorRef error;
1235  LLVMOrcMaterializationUnitRef mu;
1236 
1237  for (int i = 0; i < LookupSetSize; i++)
1238  {
1239  const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1240 
1241 #if LLVM_VERSION_MAJOR > 12
1242  LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1243 #endif
1244  symbols[i].Name = LookupSet[i].Name;
1245  symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1246  symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1247  }
1248 
1249  mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1250  error = LLVMOrcJITDylibDefine(JD, mu);
1251  if (error != LLVMErrorSuccess)
1252  LLVMOrcDisposeMaterializationUnit(mu);
1253 
1254  pfree(symbols);
1255 
1256  return error;
1257 }
1258 
1259 /*
1260  * We cannot throw errors through LLVM (without causing a FATAL at least), so
1261  * just use WARNING here. That's OK anyway, as the error is also reported at
1262  * the top level action (with less detail) and there might be multiple
1263  * invocations of errors with details.
1264  *
1265  * This doesn't really happen during normal operation, but in cases like
1266  * symbol resolution breakage. So just using elog(WARNING) is fine.
1267  */
1268 static void
1269 llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1270 {
1271  elog(WARNING, "error during JITing: %s",
1272  llvm_error_message(error));
1273 }
1274 
1275 /*
1276  * Create our own object layer, so we can add event listeners.
1277  */
1278 static LLVMOrcObjectLayerRef
1279 llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1280 {
1281  LLVMOrcObjectLayerRef objlayer =
1282  LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1283 
1284 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1286  {
1287  LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1288 
1289  LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1290  }
1291 #endif
1292 
1293 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1295  {
1296  LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1297 
1298  LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1299  }
1300 #endif
1301 
1302  return objlayer;
1303 }
1304 
1305 /*
1306  * Create LLJIT instance, using the passed in target machine. Note that the
1307  * target machine afterwards is owned by the LLJIT instance.
1308  */
1309 static LLVMOrcLLJITRef
1310 llvm_create_jit_instance(LLVMTargetMachineRef tm)
1311 {
1312  LLVMOrcLLJITRef lljit;
1313  LLVMOrcJITTargetMachineBuilderRef tm_builder;
1314  LLVMOrcLLJITBuilderRef lljit_builder;
1315  LLVMErrorRef error;
1316  LLVMOrcDefinitionGeneratorRef main_gen;
1317  LLVMOrcDefinitionGeneratorRef ref_gen;
1318 
1319  lljit_builder = LLVMOrcCreateLLJITBuilder();
1320  tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1321  LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1322 
1323  LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1324  llvm_create_object_layer,
1325  NULL);
1326 
1327  error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1328  if (error)
1329  elog(ERROR, "failed to create lljit instance: %s",
1330  llvm_error_message(error));
1331 
1332  LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1333  llvm_log_jit_error, NULL);
1334 
1335  /*
1336  * Symbol resolution support for symbols in the postgres binary /
1337  * libraries already loaded.
1338  */
1339  error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1340  LLVMOrcLLJITGetGlobalPrefix(lljit),
1341  0, NULL);
1342  if (error)
1343  elog(ERROR, "failed to create generator: %s",
1344  llvm_error_message(error));
1345  LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1346 
1347  /*
1348  * Symbol resolution support for "special" functions, e.g. a call into an
1349  * SQL callable function.
1350  */
1351 #if LLVM_VERSION_MAJOR > 14
1352  ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
1353 #else
1354  ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1355 #endif
1356  LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1357 
1358  return lljit;
1359 }
1360 
1361 static char *
1362 llvm_error_message(LLVMErrorRef error)
1363 {
1364  char *orig = LLVMGetErrorMessage(error);
1365  char *msg = pstrdup(orig);
1366 
1367  LLVMDisposeErrorMessage(orig);
1368 
1369  return msg;
1370 }
1371 
1372 #endif /* LLVM_VERSION_MAJOR > 11 */
1373 
1374 /*
1375  * ResourceOwner callbacks
1376  */
1377 static void
1379 {
1380  JitContext *context = (JitContext *) DatumGetPointer(res);
1381 
1382  context->resowner = NULL;
1383  jit_release_context(context);
1384 }
unsigned int uint32
Definition: c.h:493
void * load_external_function(const char *filename, const char *funcname, bool signalNotFound, void **filehandle)
Definition: dfmgr.c:105
int errmsg_internal(const char *fmt,...)
Definition: elog.c:1159
int errhidestmt(bool hide_stmt)
Definition: elog.c:1413
int errhidecontext(bool hide_ctx)
Definition: elog.c:1432
#define FATAL
Definition: elog.h:41
#define WARNING
Definition: elog.h:36
#define DEBUG2
Definition: elog.h:29
#define PANIC
Definition: elog.h:42
#define DEBUG1
Definition: elog.h:30
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:224
#define ereport(elevel,...)
Definition: elog.h:149
void err(int eval, const char *fmt,...)
Definition: err.c:43
void fmgr_symbol(Oid functionId, char **mod, char **fn)
Definition: fmgr.c:281
int MyProcPid
Definition: globals.c:45
char pkglib_path[MAXPGPATH]
Definition: globals.c:79
#define funcname
Definition: indent_codes.h:69
static struct @150 value
#define INSTR_TIME_SET_CURRENT(t)
Definition: instr_time.h:122
#define INSTR_TIME_GET_DOUBLE(t)
Definition: instr_time.h:188
#define INSTR_TIME_ACCUM_DIFF(x, y, z)
Definition: instr_time.h:184
void on_proc_exit(pg_on_exit_callback function, Datum arg)
Definition: ipc.c:309
bool proc_exit_inprogress
Definition: ipc.c:40
int i
Definition: isn.c:73
bool jit_dump_bitcode
Definition: jit.c:35
void jit_release_context(JitContext *context)
Definition: jit.c:137
bool jit_profiling_support
Definition: jit.c:37
bool jit_debugging_support
Definition: jit.c:34
#define PGJIT_OPT3
Definition: jit.h:21
#define PGJIT_INLINE
Definition: jit.h:22
Assert(fmt[strlen(fmt) - 1] !='\n')
List * lappend(List *list, void *datum)
Definition: list.c:339
void list_free(List *list)
Definition: list.c:1546
LLVMTypeRef StructFunctionCallInfoData
Definition: llvmjit.c:77
LLVMTypeRef StructMinimalTupleData
Definition: llvmjit.c:70
LLVMTypeRef StructMinimalTupleTableSlot
Definition: llvmjit.c:75
LLVMTypeRef StructPlanState
Definition: llvmjit.c:84
static LLVMContextRef llvm_context
Definition: llvmjit.c:102
static LLVMOrcJITStackRef llvm_opt3_orc
Definition: llvmjit.c:112
LLVMTypeRef StructExprState
Definition: llvmjit.c:80
static const char * llvm_triple
Definition: llvmjit.c:100
LLVMTypeRef StructExprEvalStep
Definition: llvmjit.c:79
LLVMValueRef llvm_pg_func(LLVMModuleRef mod, const char *funcname)
Definition: llvmjit.c:496
static void llvm_recreate_llvm_context(void)
Definition: llvmjit.c:185
struct LLVMJitHandle LLVMJitHandle
LLVMTypeRef TypeParamBool
Definition: llvmjit.c:65
LLVMTypeRef StructMemoryContextData
Definition: llvmjit.c:76
static void llvm_set_target(void)
Definition: llvmjit.c:1076
static LLVMTargetRef llvm_targetref
Definition: llvmjit.c:105
static bool llvm_session_initialized
Definition: llvmjit.c:92
LLVMTypeRef StructAggStatePerGroupData
Definition: llvmjit.c:82
static void llvm_create_types(void)
Definition: llvmjit.c:1095
LLVMTypeRef llvm_pg_var_type(const char *varname)
Definition: llvmjit.c:454
static void ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
Definition: llvmjit.c:146
static void llvm_session_initialize(void)
Definition: llvmjit.c:860
LLVMTypeRef TypeSizeT
Definition: llvmjit.c:64
static size_t llvm_jit_context_in_use_count
Definition: llvmjit.c:96
static const char * llvm_layout
Definition: llvmjit.c:101
LLVMValueRef llvm_function_reference(LLVMJitContext *context, LLVMBuilderRef builder, LLVMModuleRef mod, FunctionCallInfo fcinfo)
Definition: llvmjit.c:569
PG_MODULE_MAGIC
Definition: llvmjit.c:156
char * llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
Definition: llvmjit.c:359
static LLVMOrcJITStackRef llvm_opt0_orc
Definition: llvmjit.c:111
static void ResOwnerReleaseJitContext(Datum res)
Definition: llvmjit.c:1378
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
Definition: llvmjit.c:1149
LLVMTypeRef llvm_pg_var_func_type(const char *varname)
Definition: llvmjit.c:474
LLVMTypeRef StructTupleTableSlot
Definition: llvmjit.c:72
LLVMTypeRef TypeStorageBool
Definition: llvmjit.c:66
static LLVMTypeRef load_return_type(LLVMModuleRef mod, const char *name)
Definition: llvmjit.c:1056
static size_t llvm_generation
Definition: llvmjit.c:93
static void ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
Definition: llvmjit.c:151
static void llvm_compile_module(LLVMJitContext *context)
Definition: llvmjit.c:729
LLVMTypeRef StructHeapTupleTableSlot
Definition: llvmjit.c:74
static LLVMModuleRef llvm_types_module
Definition: llvmjit.c:90
static void llvm_shutdown(int code, Datum arg)
Definition: llvmjit.c:989
LLVMJitContext * llvm_create_context(int jitFlags)
Definition: llvmjit.c:235
static const ResourceOwnerDesc jit_resowner_desc
Definition: llvmjit.c:135
#define LLVMJIT_LLVM_CONTEXT_REUSE_MAX
Definition: llvmjit.c:48
static void llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
Definition: llvmjit.c:524
LLVMModuleRef llvm_mutable_module(LLVMJitContext *context)
Definition: llvmjit.c:334
LLVMTypeRef StructAggState
Definition: llvmjit.c:81
LLVMTypeRef TypePGFunction
Definition: llvmjit.c:67
LLVMTypeRef StructTupleDescData
Definition: llvmjit.c:71
LLVMValueRef AttributeTemplate
Definition: llvmjit.c:86
static size_t llvm_llvm_context_reuse_count
Definition: llvmjit.c:99
LLVMTypeRef StructExprContext
Definition: llvmjit.c:78
static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
Definition: llvmjit.c:632
static uint64_t llvm_resolve_symbol(const char *name, void *ctx)
Definition: llvmjit.c:1183
LLVMTypeRef StructHeapTupleHeaderData
Definition: llvmjit.c:73
LLVMTypeRef StructHeapTupleData
Definition: llvmjit.c:69
static void llvm_release_context(JitContext *context)
Definition: llvmjit.c:264
LLVMTypeRef StructNullableDatum
Definition: llvmjit.c:68
LLVMValueRef ExecEvalSubroutineTemplate
Definition: llvmjit.c:87
LLVMValueRef ExecEvalBoolSubroutineTemplate
Definition: llvmjit.c:88
void _PG_jit_provider_init(JitProviderCallbacks *cb)
Definition: llvmjit.c:163
void * llvm_get_function(LLVMJitContext *context, const char *funcname)
Definition: llvmjit.c:380
LLVMTypeRef StructAggStatePerTransData
Definition: llvmjit.c:83
void llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
Definition: llvmjit.c:548
void llvm_enter_fatal_on_oom(void)
bool llvm_in_fatal_on_oom(void)
void llvm_assert_in_fatal_section(void)
void llvm_reset_after_error(void)
void llvm_leave_fatal_on_oom(void)
bool llvm_compile_expr(ExprState *state)
Definition: llvmjit_expr.c:78
void llvm_inline(LLVMModuleRef M)
void llvm_inline_reset_caches(void)
LLVMTypeRef LLVMGetFunctionReturnType(LLVMValueRef r)
LLVMTypeRef LLVMGetFunctionType(LLVMValueRef r)
static struct pg_tm tm
Definition: localtime.c:104
char * pnstrdup(const char *in, Size len)
Definition: mcxt.c:1694
char * pstrdup(const char *in)
Definition: mcxt.c:1683
void pfree(void *pointer)
Definition: mcxt.c:1508
MemoryContext TopMemoryContext
Definition: mcxt.c:137
void * palloc0(Size size)
Definition: mcxt.c:1334
void * MemoryContextAllocZero(MemoryContext context, Size size)
Definition: mcxt.c:1202
void * MemoryContextAlloc(MemoryContext context, Size size)
Definition: mcxt.c:1168
void * palloc(Size size)
Definition: mcxt.c:1304
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:124
void * arg
#define MAXPGPATH
static char * filename
Definition: pg_dumpall.c:121
#define lfirst(lc)
Definition: pg_list.h:172
#define NIL
Definition: pg_list.h:68
static char ** options
static char * buf
Definition: pg_test_fsync.c:73
#define snprintf
Definition: port.h:238
static Datum PointerGetDatum(const void *X)
Definition: postgres.h:322
uintptr_t Datum
Definition: postgres.h:64
static Pointer DatumGetPointer(Datum X)
Definition: postgres.h:312
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
ResourceOwner CurrentResourceOwner
Definition: resowner.c:165
void ResourceOwnerForget(ResourceOwner owner, Datum value, const ResourceOwnerDesc *kind)
Definition: resowner.c:554
void ResourceOwnerRemember(ResourceOwner owner, Datum value, const ResourceOwnerDesc *kind)
Definition: resowner.c:514
void ResourceOwnerEnlarge(ResourceOwner owner)
Definition: resowner.c:442
#define RELEASE_PRIO_JIT_CONTEXTS
Definition: resowner.h:66
@ RESOURCE_RELEASE_BEFORE_LOCKS
Definition: resowner.h:54
static void error(void)
Definition: sql-dyntest.c:147
PGFunction fn_addr
Definition: fmgr.h:58
Oid fn_oid
Definition: fmgr.h:59
FmgrInfo * flinfo
Definition: fmgr.h:87
Definition: jit.h:58
ResourceOwner resowner
Definition: jit.h:62
JitProviderCompileExprCB compile_expr
Definition: jit.h:80
JitProviderResetAfterErrorCB reset_after_error
Definition: jit.h:78
JitProviderReleaseContextCB release_context
Definition: jit.h:79
LLVMOrcJITStackRef stack
Definition: llvmjit.c:57
LLVMOrcModuleHandle orc_handle
Definition: llvmjit.c:58
const char * name
Definition: resowner.h:93
Definition: type.h:95
Definition: c.h:728
const char * name