PostgreSQL Source Code  git master
llvmjit.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * llvmjit.c
4  * Core part of the LLVM JIT provider.
5  *
6  * Copyright (c) 2016-2023, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/jit/llvm/llvmjit.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres.h"
15 
16 #include <llvm-c/Analysis.h>
17 #include <llvm-c/BitReader.h>
18 #include <llvm-c/BitWriter.h>
19 #include <llvm-c/Core.h>
20 #include <llvm-c/ExecutionEngine.h>
21 #if LLVM_VERSION_MAJOR > 16
22 #include <llvm-c/Transforms/PassBuilder.h>
23 #endif
24 #if LLVM_VERSION_MAJOR > 11
25 #include <llvm-c/Orc.h>
26 #include <llvm-c/OrcEE.h>
27 #include <llvm-c/LLJIT.h>
28 #else
29 #include <llvm-c/OrcBindings.h>
30 #endif
31 #include <llvm-c/Support.h>
32 #include <llvm-c/Target.h>
33 #if LLVM_VERSION_MAJOR < 17
34 #include <llvm-c/Transforms/IPO.h>
35 #include <llvm-c/Transforms/PassManagerBuilder.h>
36 #include <llvm-c/Transforms/Scalar.h>
37 #if LLVM_VERSION_MAJOR > 6
38 #include <llvm-c/Transforms/Utils.h>
39 #endif
40 #endif
41 
42 #include "jit/llvmjit.h"
43 #include "jit/llvmjit_emit.h"
44 #include "miscadmin.h"
45 #include "portability/instr_time.h"
46 #include "storage/ipc.h"
47 #include "utils/memutils.h"
48 #include "utils/resowner.h"
49 
50 #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
51 
52 /* Handle of a module emitted via ORC JIT */
53 typedef struct LLVMJitHandle
54 {
55 #if LLVM_VERSION_MAJOR > 11
56  LLVMOrcLLJITRef lljit;
57  LLVMOrcResourceTrackerRef resource_tracker;
58 #else
59  LLVMOrcJITStackRef stack;
60  LLVMOrcModuleHandle orc_handle;
61 #endif
63 
64 
65 /* types & functions commonly needed for JITing */
66 LLVMTypeRef TypeSizeT;
67 LLVMTypeRef TypeParamBool;
68 LLVMTypeRef TypeStorageBool;
69 LLVMTypeRef TypePGFunction;
70 LLVMTypeRef StructNullableDatum;
71 LLVMTypeRef StructHeapTupleData;
73 LLVMTypeRef StructTupleDescData;
80 LLVMTypeRef StructExprContext;
81 LLVMTypeRef StructExprEvalStep;
82 LLVMTypeRef StructExprState;
83 LLVMTypeRef StructAggState;
86 LLVMTypeRef StructPlanState;
87 
88 LLVMValueRef AttributeTemplate;
91 
92 static LLVMModuleRef llvm_types_module = NULL;
93 
94 static bool llvm_session_initialized = false;
95 static size_t llvm_generation = 0;
96 
97 /* number of LLVMJitContexts that currently are in use */
99 
100 /* how many times has the current LLVMContextRef been used */
102 static const char *llvm_triple = NULL;
103 static const char *llvm_layout = NULL;
104 static LLVMContextRef llvm_context;
105 
106 
107 static LLVMTargetRef llvm_targetref;
108 #if LLVM_VERSION_MAJOR > 11
109 static LLVMOrcThreadSafeContextRef llvm_ts_context;
110 static LLVMOrcLLJITRef llvm_opt0_orc;
111 static LLVMOrcLLJITRef llvm_opt3_orc;
112 #else /* LLVM_VERSION_MAJOR > 11 */
113 static LLVMOrcJITStackRef llvm_opt0_orc;
114 static LLVMOrcJITStackRef llvm_opt3_orc;
115 #endif /* LLVM_VERSION_MAJOR > 11 */
116 
117 
118 static void llvm_release_context(JitContext *context);
119 static void llvm_session_initialize(void);
120 static void llvm_shutdown(int code, Datum arg);
121 static void llvm_compile_module(LLVMJitContext *context);
122 static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
123 
124 static void llvm_create_types(void);
125 static void llvm_set_target(void);
126 static void llvm_recreate_llvm_context(void);
127 static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
128 
129 #if LLVM_VERSION_MAJOR > 11
130 static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
131 static char *llvm_error_message(LLVMErrorRef error);
132 #endif /* LLVM_VERSION_MAJOR > 11 */
133 
134 /* ResourceOwner callbacks to hold JitContexts */
135 static void ResOwnerReleaseJitContext(Datum res);
136 
138 {
139  .name = "LLVM JIT context",
140  .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
141  .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
142  .ReleaseResource = ResOwnerReleaseJitContext,
143  .DebugPrint = NULL /* the default message is fine */
144 };
145 
146 /* Convenience wrappers over ResourceOwnerRemember/Forget */
147 static inline void
148 ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
149 {
151 }
152 static inline void
153 ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
154 {
156 }
157 
159 
160 
161 /*
162  * Initialize LLVM JIT provider.
163  */
164 void
166 {
170 }
171 
172 
173 /*
174  * Every now and then create a new LLVMContextRef. Unfortunately, during every
175  * round of inlining, types may "leak" (they can still be found/used via the
176  * context, but new types will be created the next time in inlining is
177  * performed). To prevent that from slowly accumulating problematic amounts of
178  * memory, recreate the LLVMContextRef we use. We don't want to do so too
179  * often, as that implies some overhead (particularly re-loading the module
180  * summaries / modules is fairly expensive). A future TODO would be to make
181  * this more finegrained and only drop/recreate the LLVMContextRef when we know
182  * there has been inlining. If we can get the size of the context from LLVM
183  * then that might be a better way to determine when to drop/recreate rather
184  * then the usagecount heuristic currently employed.
185  */
186 static void
188 {
189  if (!llvm_context)
190  elog(ERROR, "Trying to recreate a non-existing context");
191 
192  /*
193  * We can only safely recreate the LLVM context if no other code is being
194  * JITed, otherwise we'd release the types in use for that.
195  */
197  {
199  return;
200  }
201 
203  {
205  return;
206  }
207 
208  /*
209  * Need to reset the modules that the inlining code caches before
210  * disposing of the context. LLVM modules exist within a specific LLVM
211  * context, therefore disposing of the context before resetting the cache
212  * would lead to dangling pointers to modules.
213  */
215 
216  LLVMContextDispose(llvm_context);
217  llvm_context = LLVMContextCreate();
219 
220  /*
221  * Re-build cached type information, so code generation code can rely on
222  * that information to be present (also prevents the variables to be
223  * dangling references).
224  */
226 }
227 
228 
229 /*
230  * Create a context for JITing work.
231  *
232  * The context, including subsidiary resources, will be cleaned up either when
233  * the context is explicitly released, or when the lifetime of
234  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
235  */
236 LLVMJitContext *
237 llvm_create_context(int jitFlags)
238 {
239  LLVMJitContext *context;
240 
242 
244 
246 
248 
250  sizeof(LLVMJitContext));
251  context->base.flags = jitFlags;
252 
253  /* ensure cleanup */
254  context->base.resowner = CurrentResourceOwner;
256 
258 
259  return context;
260 }
261 
262 /*
263  * Release resources required by one llvm context.
264  */
265 static void
267 {
268  LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
269  ListCell *lc;
270 
271  /*
272  * Consider as cleaned up even if we skip doing so below, that way we can
273  * verify the tracking is correct (see llvm_shutdown()).
274  */
276 
277  /*
278  * When this backend is exiting, don't clean up LLVM. As an error might
279  * have occurred from within LLVM, we do not want to risk reentering. All
280  * resource cleanup is going to happen through process exit.
281  */
283  return;
284 
286 
287  if (llvm_jit_context->module)
288  {
289  LLVMDisposeModule(llvm_jit_context->module);
290  llvm_jit_context->module = NULL;
291  }
292 
293  foreach(lc, llvm_jit_context->handles)
294  {
295  LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
296 
297 #if LLVM_VERSION_MAJOR > 11
298  {
299  LLVMOrcExecutionSessionRef ee;
300  LLVMOrcSymbolStringPoolRef sp;
301 
302  LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
303  LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
304 
305  /*
306  * Without triggering cleanup of the string pool, we'd leak
307  * memory. It'd be sufficient to do this far less often, but in
308  * experiments the required time was small enough to just always
309  * do it.
310  */
311  ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
312  sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
313  LLVMOrcSymbolStringPoolClearDeadEntries(sp);
314  }
315 #else /* LLVM_VERSION_MAJOR > 11 */
316  {
317  LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
318  }
319 #endif /* LLVM_VERSION_MAJOR > 11 */
320 
321  pfree(jit_handle);
322  }
323  list_free(llvm_jit_context->handles);
324  llvm_jit_context->handles = NIL;
325 
327 
328  if (context->resowner)
329  ResourceOwnerForgetJIT(context->resowner, llvm_jit_context);
330 }
331 
332 /*
333  * Return module which may be modified, e.g. by creating new functions.
334  */
335 LLVMModuleRef
336 llvm_mutable_module(LLVMJitContext *context)
337 {
339 
340  /*
341  * If there's no in-progress module, create a new one.
342  */
343  if (!context->module)
344  {
345  context->compiled = false;
346  context->module_generation = llvm_generation++;
347  context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
348  LLVMSetTarget(context->module, llvm_triple);
349  LLVMSetDataLayout(context->module, llvm_layout);
350  }
351 
352  return context->module;
353 }
354 
355 /*
356  * Expand function name to be non-conflicting. This should be used by code
357  * generating code, when adding new externally visible function definitions to
358  * a Module.
359  */
360 char *
361 llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
362 {
363  Assert(context->module != NULL);
364 
365  context->base.instr.created_functions++;
366 
367  /*
368  * Previously we used dots to separate, but turns out some tools, e.g.
369  * GDB, don't like that and truncate name.
370  */
371  return psprintf("%s_%zu_%d",
372  basename,
373  context->module_generation,
374  context->counter++);
375 }
376 
377 /*
378  * Return pointer to function funcname, which has to exist. If there's pending
379  * code to be optimized and emitted, do so first.
380  */
381 void *
382 llvm_get_function(LLVMJitContext *context, const char *funcname)
383 {
384 #if LLVM_VERSION_MAJOR > 11 || \
385  defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
386  ListCell *lc;
387 #endif
388 
390 
391  /*
392  * If there is a pending / not emitted module, compile and emit now.
393  * Otherwise we might not find the [correct] function.
394  */
395  if (!context->compiled)
396  {
397  llvm_compile_module(context);
398  }
399 
400  /*
401  * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
402  * to mangle here.
403  */
404 
405 #if LLVM_VERSION_MAJOR > 11
406  foreach(lc, context->handles)
407  {
408  LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
409  instr_time starttime;
410  instr_time endtime;
411  LLVMErrorRef error;
412  LLVMOrcJITTargetAddress addr;
413 
414  INSTR_TIME_SET_CURRENT(starttime);
415 
416  addr = 0;
417  error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
418  if (error)
419  elog(ERROR, "failed to look up symbol \"%s\": %s",
420  funcname, llvm_error_message(error));
421 
422  /*
423  * LLJIT only actually emits code the first time a symbol is
424  * referenced. Thus add lookup time to emission time. That's counting
425  * a bit more than with older LLVM versions, but unlikely to ever
426  * matter.
427  */
428  INSTR_TIME_SET_CURRENT(endtime);
429  INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
430  endtime, starttime);
431 
432  if (addr)
433  return (void *) (uintptr_t) addr;
434  }
435 #elif defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
436  foreach(lc, context->handles)
437  {
438  LLVMOrcTargetAddress addr;
439  LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
440 
441  addr = 0;
442  if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
443  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
444  if (addr)
445  return (void *) (uintptr_t) addr;
446  }
447 #elif LLVM_VERSION_MAJOR < 5
448  {
449  LLVMOrcTargetAddress addr;
450 
451  if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
452  return (void *) (uintptr_t) addr;
453  if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
454  return (void *) (uintptr_t) addr;
455  }
456 #else
457  {
458  LLVMOrcTargetAddress addr;
459 
460  if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
461  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
462  if (addr)
463  return (void *) (uintptr_t) addr;
464  if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
465  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
466  if (addr)
467  return (void *) (uintptr_t) addr;
468  }
469 #endif
470 
471  elog(ERROR, "failed to JIT: %s", funcname);
472 
473  return NULL;
474 }
475 
476 /*
477  * Return type of a variable in llvmjit_types.c. This is useful to keep types
478  * in sync between plain C and JIT related code.
479  */
480 LLVMTypeRef
481 llvm_pg_var_type(const char *varname)
482 {
483  LLVMValueRef v_srcvar;
484  LLVMTypeRef typ;
485 
486  /* this'll return a *pointer* to the global */
487  v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
488  if (!v_srcvar)
489  elog(ERROR, "variable %s not in llvmjit_types.c", varname);
490 
491  typ = LLVMGlobalGetValueType(v_srcvar);
492 
493  return typ;
494 }
495 
496 /*
497  * Return function type of a variable in llvmjit_types.c. This is useful to
498  * keep function types in sync between C and JITed code.
499  */
500 LLVMTypeRef
501 llvm_pg_var_func_type(const char *varname)
502 {
503  LLVMValueRef v_srcvar;
504  LLVMTypeRef typ;
505 
506  v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
507  if (!v_srcvar)
508  elog(ERROR, "function %s not in llvmjit_types.c", varname);
509 
510  typ = LLVMGetFunctionType(v_srcvar);
511 
512  return typ;
513 }
514 
515 /*
516  * Return declaration for a function referenced in llvmjit_types.c, adding it
517  * to the module if necessary.
518  *
519  * This is used to make functions discovered via llvm_create_types() known to
520  * the module that's currently being worked on.
521  */
522 LLVMValueRef
523 llvm_pg_func(LLVMModuleRef mod, const char *funcname)
524 {
525  LLVMValueRef v_srcfn;
526  LLVMValueRef v_fn;
527 
528  /* don't repeatedly add function */
529  v_fn = LLVMGetNamedFunction(mod, funcname);
530  if (v_fn)
531  return v_fn;
532 
533  v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
534 
535  if (!v_srcfn)
536  elog(ERROR, "function %s not in llvmjit_types.c", funcname);
537 
538  v_fn = LLVMAddFunction(mod,
539  funcname,
540  LLVMGetFunctionType(v_srcfn));
541  llvm_copy_attributes(v_srcfn, v_fn);
542 
543  return v_fn;
544 }
545 
546 /*
547  * Copy attributes from one function to another, for a specific index (an
548  * index can reference return value, function and parameter attributes).
549  */
550 static void
551 llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
552 {
553  int num_attributes;
554  LLVMAttributeRef *attrs;
555 
556  num_attributes = LLVMGetAttributeCountAtIndexPG(v_from, index);
557 
558  /*
559  * Not just for efficiency: LLVM <= 3.9 crashes when
560  * LLVMGetAttributesAtIndex() is called for an index with 0 attributes.
561  */
562  if (num_attributes == 0)
563  return;
564 
565  attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
566  LLVMGetAttributesAtIndex(v_from, index, attrs);
567 
568  for (int attno = 0; attno < num_attributes; attno++)
569  LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
570 
571  pfree(attrs);
572 }
573 
574 /*
575  * Copy all attributes from one function to another. I.e. function, return and
576  * parameters will be copied.
577  */
578 void
579 llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
580 {
581  uint32 param_count;
582 
583  /* copy function attributes */
584  llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
585 
586  /* and the return value attributes */
587  llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
588 
589  /* and each function parameter's attribute */
590  param_count = LLVMCountParams(v_from);
591 
592  for (int paramidx = 1; paramidx <= param_count; paramidx++)
593  llvm_copy_attributes_at_index(v_from, v_to, paramidx);
594 }
595 
596 /*
597  * Return a callable LLVMValueRef for fcinfo.
598  */
599 LLVMValueRef
600 llvm_function_reference(LLVMJitContext *context,
601  LLVMBuilderRef builder,
602  LLVMModuleRef mod,
603  FunctionCallInfo fcinfo)
604 {
605  char *modname;
606  char *basename;
607  char *funcname;
608 
609  LLVMValueRef v_fn;
610 
611  fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
612 
613  if (modname != NULL && basename != NULL)
614  {
615  /* external function in loadable library */
616  funcname = psprintf("pgextern.%s.%s", modname, basename);
617  }
618  else if (basename != NULL)
619  {
620  /* internal function */
621  funcname = pstrdup(basename);
622  }
623  else
624  {
625  /*
626  * Function we don't know to handle, return pointer. We do so by
627  * creating a global constant containing a pointer to the function.
628  * Makes IR more readable.
629  */
630  LLVMValueRef v_fn_addr;
631 
632  funcname = psprintf("pgoidextern.%u",
633  fcinfo->flinfo->fn_oid);
634  v_fn = LLVMGetNamedGlobal(mod, funcname);
635  if (v_fn != 0)
636  return l_load(builder, TypePGFunction, v_fn, "");
637 
638  v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
639 
640  v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
641  LLVMSetInitializer(v_fn, v_fn_addr);
642  LLVMSetGlobalConstant(v_fn, true);
643  LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
644  LLVMSetUnnamedAddr(v_fn, true);
645 
646  return l_load(builder, TypePGFunction, v_fn, "");
647  }
648 
649  /* check if function already has been added */
650  v_fn = LLVMGetNamedFunction(mod, funcname);
651  if (v_fn != 0)
652  return v_fn;
653 
654  v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
655 
656  return v_fn;
657 }
658 
659 /*
660  * Optimize code in module using the flags set in context.
661  */
662 static void
663 llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
664 {
665 #if LLVM_VERSION_MAJOR < 17
666  LLVMPassManagerBuilderRef llvm_pmb;
667  LLVMPassManagerRef llvm_mpm;
668  LLVMPassManagerRef llvm_fpm;
669  LLVMValueRef func;
670  int compile_optlevel;
671 
672  if (context->base.flags & PGJIT_OPT3)
673  compile_optlevel = 3;
674  else
675  compile_optlevel = 0;
676 
677  /*
678  * Have to create a new pass manager builder every pass through, as the
679  * inliner has some per-builder state. Otherwise one ends up only inlining
680  * a function the first time though.
681  */
682  llvm_pmb = LLVMPassManagerBuilderCreate();
683  LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
684  llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
685 
686  if (context->base.flags & PGJIT_OPT3)
687  {
688  /* TODO: Unscientifically determined threshold */
689  LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
690  }
691  else
692  {
693  /* we rely on mem2reg heavily, so emit even in the O0 case */
694  LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
695  }
696 
697  LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
698 
699  /*
700  * Do function level optimization. This could be moved to the point where
701  * functions are emitted, to reduce memory usage a bit.
702  */
703  LLVMInitializeFunctionPassManager(llvm_fpm);
704  for (func = LLVMGetFirstFunction(context->module);
705  func != NULL;
706  func = LLVMGetNextFunction(func))
707  LLVMRunFunctionPassManager(llvm_fpm, func);
708  LLVMFinalizeFunctionPassManager(llvm_fpm);
709  LLVMDisposePassManager(llvm_fpm);
710 
711  /*
712  * Perform module level optimization. We do so even in the non-optimized
713  * case, so always-inline functions etc get inlined. It's cheap enough.
714  */
715  llvm_mpm = LLVMCreatePassManager();
716  LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
717  llvm_mpm);
718  /* always use always-inliner pass */
719  if (!(context->base.flags & PGJIT_OPT3))
720  LLVMAddAlwaysInlinerPass(llvm_mpm);
721  /* if doing inlining, but no expensive optimization, add inlining pass */
722  if (context->base.flags & PGJIT_INLINE
723  && !(context->base.flags & PGJIT_OPT3))
724  LLVMAddFunctionInliningPass(llvm_mpm);
725  LLVMRunPassManager(llvm_mpm, context->module);
726  LLVMDisposePassManager(llvm_mpm);
727 
728  LLVMPassManagerBuilderDispose(llvm_pmb);
729 #else
730  LLVMPassBuilderOptionsRef options;
731  LLVMErrorRef err;
732  const char *passes;
733 
734  if (context->base.flags & PGJIT_OPT3)
735  passes = "default<O3>";
736  else
737  passes = "default<O0>,mem2reg";
738 
739  options = LLVMCreatePassBuilderOptions();
740 
741 #ifdef LLVM_PASS_DEBUG
742  LLVMPassBuilderOptionsSetDebugLogging(options, 1);
743 #endif
744 
745  LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
746 
747  err = LLVMRunPasses(module, passes, NULL, options);
748 
749  if (err)
750  elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
751 
752  LLVMDisposePassBuilderOptions(options);
753 #endif
754 }
755 
756 /*
757  * Emit code for the currently pending module.
758  */
759 static void
760 llvm_compile_module(LLVMJitContext *context)
761 {
762  LLVMJitHandle *handle;
763  MemoryContext oldcontext;
764  instr_time starttime;
765  instr_time endtime;
766 #if LLVM_VERSION_MAJOR > 11
767  LLVMOrcLLJITRef compile_orc;
768 #else
769  LLVMOrcJITStackRef compile_orc;
770 #endif
771 
772  if (context->base.flags & PGJIT_OPT3)
773  compile_orc = llvm_opt3_orc;
774  else
775  compile_orc = llvm_opt0_orc;
776 
777  /* perform inlining */
778  if (context->base.flags & PGJIT_INLINE)
779  {
780  INSTR_TIME_SET_CURRENT(starttime);
781  llvm_inline(context->module);
782  INSTR_TIME_SET_CURRENT(endtime);
783  INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
784  endtime, starttime);
785  }
786 
787  if (jit_dump_bitcode)
788  {
789  char *filename;
790 
791  filename = psprintf("%d.%zu.bc",
792  MyProcPid,
793  context->module_generation);
794  LLVMWriteBitcodeToFile(context->module, filename);
795  pfree(filename);
796  }
797 
798 
799  /* optimize according to the chosen optimization settings */
800  INSTR_TIME_SET_CURRENT(starttime);
801  llvm_optimize_module(context, context->module);
802  INSTR_TIME_SET_CURRENT(endtime);
803  INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
804  endtime, starttime);
805 
806  if (jit_dump_bitcode)
807  {
808  char *filename;
809 
810  filename = psprintf("%d.%zu.optimized.bc",
811  MyProcPid,
812  context->module_generation);
813  LLVMWriteBitcodeToFile(context->module, filename);
814  pfree(filename);
815  }
816 
817  handle = (LLVMJitHandle *)
819 
820  /*
821  * Emit the code. Note that this can, depending on the optimization
822  * settings, take noticeable resources as code emission executes low-level
823  * instruction combining/selection passes etc. Without optimization a
824  * faster instruction selection mechanism is used.
825  */
826  INSTR_TIME_SET_CURRENT(starttime);
827 #if LLVM_VERSION_MAJOR > 11
828  {
829  LLVMOrcThreadSafeModuleRef ts_module;
830  LLVMErrorRef error;
831  LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
832 
833  ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
834 
835  handle->lljit = compile_orc;
836  handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
837 
838  /*
839  * NB: This doesn't actually emit code. That happens lazily the first
840  * time a symbol defined in the module is requested. Due to that
841  * llvm_get_function() also accounts for emission time.
842  */
843 
844  context->module = NULL; /* will be owned by LLJIT */
845  error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
846  handle->resource_tracker,
847  ts_module);
848 
849  if (error)
850  elog(ERROR, "failed to JIT module: %s",
851  llvm_error_message(error));
852 
853  /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
854  }
855 #elif LLVM_VERSION_MAJOR > 6
856  {
857  handle->stack = compile_orc;
858  if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
859  llvm_resolve_symbol, NULL))
860  elog(ERROR, "failed to JIT module");
861 
862  /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
863  }
864 #elif LLVM_VERSION_MAJOR > 4
865  {
866  LLVMSharedModuleRef smod;
867 
868  smod = LLVMOrcMakeSharedModule(context->module);
869  handle->stack = compile_orc;
870  if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, smod,
871  llvm_resolve_symbol, NULL))
872  elog(ERROR, "failed to JIT module");
873 
874  LLVMOrcDisposeSharedModuleRef(smod);
875  }
876 #else /* LLVM 4.0 and 3.9 */
877  {
878  handle->stack = compile_orc;
879  handle->orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
880  llvm_resolve_symbol, NULL);
881 
882  LLVMDisposeModule(context->module);
883  }
884 #endif
885 
886  INSTR_TIME_SET_CURRENT(endtime);
887  INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
888  endtime, starttime);
889 
890  context->module = NULL;
891  context->compiled = true;
892 
893  /* remember emitted code for cleanup and lookups */
895  context->handles = lappend(context->handles, handle);
896  MemoryContextSwitchTo(oldcontext);
897 
898  ereport(DEBUG1,
899  (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
900  INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
901  INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
902  INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
903  errhidestmt(true),
904  errhidecontext(true)));
905 }
906 
907 /*
908  * Per session initialization.
909  */
910 static void
912 {
913  MemoryContext oldcontext;
914  char *error = NULL;
915  char *cpu = NULL;
916  char *features = NULL;
917  LLVMTargetMachineRef opt0_tm;
918  LLVMTargetMachineRef opt3_tm;
919 
921  return;
922 
924 
925  LLVMInitializeNativeTarget();
926  LLVMInitializeNativeAsmPrinter();
927  LLVMInitializeNativeAsmParser();
928 
929  if (llvm_context == NULL)
930  {
931  llvm_context = LLVMContextCreate();
932 
935  }
936 
937  /*
938  * When targeting LLVM 15, turn off opaque pointers for the context we
939  * build our code in. We don't need to do so for other contexts (e.g.
940  * llvm_ts_context). Once the IR is generated, it carries the necessary
941  * information.
942  *
943  * For 16 and above, opaque pointers must be used, and we have special
944  * code for that.
945  */
946 #if LLVM_VERSION_MAJOR == 15
947  LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
948 #endif
949 
950  /*
951  * Synchronize types early, as that also includes inferring the target
952  * triple.
953  */
955 
956  /*
957  * Extract target information from loaded module.
958  */
959  llvm_set_target();
960 
961  if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
962  {
963  elog(FATAL, "failed to query triple %s", error);
964  }
965 
966  /*
967  * We want the generated code to use all available features. Therefore
968  * grab the host CPU string and detect features of the current CPU. The
969  * latter is needed because some CPU architectures default to enabling
970  * features not all CPUs have (weird, huh).
971  */
972  cpu = LLVMGetHostCPUName();
973  features = LLVMGetHostCPUFeatures();
974  elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
975  cpu, features);
976 
977  opt0_tm =
978  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
979  LLVMCodeGenLevelNone,
980  LLVMRelocDefault,
981  LLVMCodeModelJITDefault);
982  opt3_tm =
983  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
984  LLVMCodeGenLevelAggressive,
985  LLVMRelocDefault,
986  LLVMCodeModelJITDefault);
987 
988  LLVMDisposeMessage(cpu);
989  cpu = NULL;
990  LLVMDisposeMessage(features);
991  features = NULL;
992 
993  /* force symbols in main binary to be loaded */
994  LLVMLoadLibraryPermanently(NULL);
995 
996 #if LLVM_VERSION_MAJOR > 11
997  {
998  llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
999 
1000  llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
1001  opt0_tm = 0;
1002 
1003  llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
1004  opt3_tm = 0;
1005  }
1006 #else /* LLVM_VERSION_MAJOR > 11 */
1007  {
1008  llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
1009  llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
1010 
1011 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1013  {
1014  LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1015 
1016  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
1017  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
1018  }
1019 #endif
1020 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1022  {
1023  LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1024 
1025  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
1026  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
1027  }
1028 #endif
1029  }
1030 #endif /* LLVM_VERSION_MAJOR > 11 */
1031 
1033 
1034  llvm_session_initialized = true;
1035 
1036  MemoryContextSwitchTo(oldcontext);
1037 }
1038 
1039 static void
1041 {
1042  /*
1043  * If llvm_shutdown() is reached while in a fatal-on-oom section an error
1044  * has occurred in the middle of LLVM code. It is not safe to call back
1045  * into LLVM (which is why a FATAL error was thrown).
1046  *
1047  * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
1048  * profiling data won't be written out.
1049  */
1050  if (llvm_in_fatal_on_oom())
1051  {
1053  return;
1054  }
1055 
1057  elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
1059 
1060 #if LLVM_VERSION_MAJOR > 11
1061  {
1062  if (llvm_opt3_orc)
1063  {
1064  LLVMOrcDisposeLLJIT(llvm_opt3_orc);
1065  llvm_opt3_orc = NULL;
1066  }
1067  if (llvm_opt0_orc)
1068  {
1069  LLVMOrcDisposeLLJIT(llvm_opt0_orc);
1070  llvm_opt0_orc = NULL;
1071  }
1072  if (llvm_ts_context)
1073  {
1074  LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
1075  llvm_ts_context = NULL;
1076  }
1077  }
1078 #else /* LLVM_VERSION_MAJOR > 11 */
1079  {
1080  /* unregister profiling support, needs to be flushed to be useful */
1081 
1082  if (llvm_opt3_orc)
1083  {
1084 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
1086  LLVMOrcUnregisterPerf(llvm_opt3_orc);
1087 #endif
1088  LLVMOrcDisposeInstance(llvm_opt3_orc);
1089  llvm_opt3_orc = NULL;
1090  }
1091 
1092  if (llvm_opt0_orc)
1093  {
1094 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
1096  LLVMOrcUnregisterPerf(llvm_opt0_orc);
1097 #endif
1098  LLVMOrcDisposeInstance(llvm_opt0_orc);
1099  llvm_opt0_orc = NULL;
1100  }
1101  }
1102 #endif /* LLVM_VERSION_MAJOR > 11 */
1103 }
1104 
1105 /* helper for llvm_create_types, returning a function's return type */
1106 static LLVMTypeRef
1107 load_return_type(LLVMModuleRef mod, const char *name)
1108 {
1109  LLVMValueRef value;
1110  LLVMTypeRef typ;
1111 
1112  /* this'll return a *pointer* to the function */
1113  value = LLVMGetNamedFunction(mod, name);
1114  if (!value)
1115  elog(ERROR, "function %s is unknown", name);
1116 
1117  typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
1118 
1119  return typ;
1120 }
1121 
1122 /*
1123  * Load triple & layout from clang emitted file so we're guaranteed to be
1124  * compatible.
1125  */
1126 static void
1128 {
1129  if (!llvm_types_module)
1130  elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
1131 
1132  if (llvm_triple == NULL)
1133  llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
1134 
1135  if (llvm_layout == NULL)
1136  llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
1137 }
1138 
1139 /*
1140  * Load required information, types, function signatures from llvmjit_types.c
1141  * and make them available in global variables.
1142  *
1143  * Those global variables are then used while emitting code.
1144  */
1145 static void
1147 {
1148  char path[MAXPGPATH];
1149  LLVMMemoryBufferRef buf;
1150  char *msg;
1151 
1152  snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
1153 
1154  /* open file */
1155  if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
1156  {
1157  elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
1158  path, msg);
1159  }
1160 
1161  /* eagerly load contents, going to need it all */
1162  if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
1163  {
1164  elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
1165  }
1166  LLVMDisposeMemoryBuffer(buf);
1167 
1168  TypeSizeT = llvm_pg_var_type("TypeSizeT");
1169  TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
1170  TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
1171  TypePGFunction = llvm_pg_var_type("TypePGFunction");
1172  StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
1173  StructExprContext = llvm_pg_var_type("StructExprContext");
1174  StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
1175  StructExprState = llvm_pg_var_type("StructExprState");
1176  StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
1177  StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
1178  StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
1179  StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
1180  StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
1181  StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
1182  StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
1183  StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
1184  StructAggState = llvm_pg_var_type("StructAggState");
1185  StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
1186  StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
1187  StructPlanState = llvm_pg_var_type("StructPlanState");
1188  StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
1189 
1190  AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
1191  ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
1192  ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
1193 }
1194 
1195 /*
1196  * Split a symbol into module / function parts. If the function is in the
1197  * main binary (or an external library) *modname will be NULL.
1198  */
1199 void
1200 llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1201 {
1202  *modname = NULL;
1203  *funcname = NULL;
1204 
1205  /*
1206  * Module function names are pgextern.$module.$funcname
1207  */
1208  if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1209  {
1210  /*
1211  * Symbol names cannot contain a ., therefore we can split based on
1212  * first and last occurrence of one.
1213  */
1214  *funcname = rindex(name, '.');
1215  (*funcname)++; /* jump over . */
1216 
1217  *modname = pnstrdup(name + strlen("pgextern."),
1218  *funcname - name - strlen("pgextern.") - 1);
1219  Assert(funcname);
1220 
1221  *funcname = pstrdup(*funcname);
1222  }
1223  else
1224  {
1225  *modname = NULL;
1226  *funcname = pstrdup(name);
1227  }
1228 }
1229 
1230 /*
1231  * Attempt to resolve symbol, so LLVM can emit a reference to it.
1232  */
1233 static uint64_t
1234 llvm_resolve_symbol(const char *symname, void *ctx)
1235 {
1236  uintptr_t addr;
1237  char *funcname;
1238  char *modname;
1239 
1240  /*
1241  * macOS prefixes all object level symbols with an underscore. But neither
1242  * dlsym() nor PG's inliner expect that. So undo.
1243  */
1244 #if defined(__darwin__)
1245  if (symname[0] != '_')
1246  elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1247  symname++;
1248 #endif
1249 
1250  llvm_split_symbol_name(symname, &modname, &funcname);
1251 
1252  /* functions that aren't resolved to names shouldn't ever get here */
1253  Assert(funcname);
1254 
1255  if (modname)
1256  addr = (uintptr_t) load_external_function(modname, funcname,
1257  true, NULL);
1258  else
1259  addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1260 
1261  pfree(funcname);
1262  if (modname)
1263  pfree(modname);
1264 
1265  /* let LLVM will error out - should never happen */
1266  if (!addr)
1267  elog(WARNING, "failed to resolve name %s", symname);
1268 
1269  return (uint64_t) addr;
1270 }
1271 
1272 #if LLVM_VERSION_MAJOR > 11
1273 
1274 static LLVMErrorRef
1275 llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1276  LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1277  LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1278  LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1279 {
1280 #if LLVM_VERSION_MAJOR > 14
1281  LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
1282 #else
1283  LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1284 #endif
1285  LLVMErrorRef error;
1286  LLVMOrcMaterializationUnitRef mu;
1287 
1288  for (int i = 0; i < LookupSetSize; i++)
1289  {
1290  const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1291 
1292 #if LLVM_VERSION_MAJOR > 12
1293  LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1294 #endif
1295  symbols[i].Name = LookupSet[i].Name;
1296  symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1297  symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1298  }
1299 
1300  mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1301  error = LLVMOrcJITDylibDefine(JD, mu);
1302  if (error != LLVMErrorSuccess)
1303  LLVMOrcDisposeMaterializationUnit(mu);
1304 
1305  pfree(symbols);
1306 
1307  return error;
1308 }
1309 
1310 /*
1311  * We cannot throw errors through LLVM (without causing a FATAL at least), so
1312  * just use WARNING here. That's OK anyway, as the error is also reported at
1313  * the top level action (with less detail) and there might be multiple
1314  * invocations of errors with details.
1315  *
1316  * This doesn't really happen during normal operation, but in cases like
1317  * symbol resolution breakage. So just using elog(WARNING) is fine.
1318  */
1319 static void
1320 llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1321 {
1322  elog(WARNING, "error during JITing: %s",
1323  llvm_error_message(error));
1324 }
1325 
1326 /*
1327  * Create our own object layer, so we can add event listeners.
1328  */
1329 static LLVMOrcObjectLayerRef
1330 llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1331 {
1332  LLVMOrcObjectLayerRef objlayer =
1333  LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1334 
1335 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1337  {
1338  LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1339 
1340  LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1341  }
1342 #endif
1343 
1344 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1346  {
1347  LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1348 
1349  LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1350  }
1351 #endif
1352 
1353  return objlayer;
1354 }
1355 
1356 /*
1357  * Create LLJIT instance, using the passed in target machine. Note that the
1358  * target machine afterwards is owned by the LLJIT instance.
1359  */
1360 static LLVMOrcLLJITRef
1361 llvm_create_jit_instance(LLVMTargetMachineRef tm)
1362 {
1363  LLVMOrcLLJITRef lljit;
1364  LLVMOrcJITTargetMachineBuilderRef tm_builder;
1365  LLVMOrcLLJITBuilderRef lljit_builder;
1366  LLVMErrorRef error;
1367  LLVMOrcDefinitionGeneratorRef main_gen;
1368  LLVMOrcDefinitionGeneratorRef ref_gen;
1369 
1370  lljit_builder = LLVMOrcCreateLLJITBuilder();
1371  tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1372  LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1373 
1374  LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1375  llvm_create_object_layer,
1376  NULL);
1377 
1378  error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1379  if (error)
1380  elog(ERROR, "failed to create lljit instance: %s",
1381  llvm_error_message(error));
1382 
1383  LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1384  llvm_log_jit_error, NULL);
1385 
1386  /*
1387  * Symbol resolution support for symbols in the postgres binary /
1388  * libraries already loaded.
1389  */
1390  error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1391  LLVMOrcLLJITGetGlobalPrefix(lljit),
1392  0, NULL);
1393  if (error)
1394  elog(ERROR, "failed to create generator: %s",
1395  llvm_error_message(error));
1396  LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1397 
1398  /*
1399  * Symbol resolution support for "special" functions, e.g. a call into an
1400  * SQL callable function.
1401  */
1402 #if LLVM_VERSION_MAJOR > 14
1403  ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
1404 #else
1405  ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1406 #endif
1407  LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1408 
1409  return lljit;
1410 }
1411 
1412 static char *
1413 llvm_error_message(LLVMErrorRef error)
1414 {
1415  char *orig = LLVMGetErrorMessage(error);
1416  char *msg = pstrdup(orig);
1417 
1418  LLVMDisposeErrorMessage(orig);
1419 
1420  return msg;
1421 }
1422 
1423 #endif /* LLVM_VERSION_MAJOR > 11 */
1424 
1425 /*
1426  * ResourceOwner callbacks
1427  */
1428 static void
1430 {
1431  JitContext *context = (JitContext *) DatumGetPointer(res);
1432 
1433  context->resowner = NULL;
1434  jit_release_context(context);
1435 }
unsigned int uint32
Definition: c.h:495
void * load_external_function(const char *filename, const char *funcname, bool signalNotFound, void **filehandle)
Definition: dfmgr.c:105
int errmsg_internal(const char *fmt,...)
Definition: elog.c:1156
int errhidestmt(bool hide_stmt)
Definition: elog.c:1410
int errhidecontext(bool hide_ctx)
Definition: elog.c:1429
#define FATAL
Definition: elog.h:41
#define WARNING
Definition: elog.h:36
#define DEBUG2
Definition: elog.h:29
#define PANIC
Definition: elog.h:42
#define DEBUG1
Definition: elog.h:30
#define ERROR
Definition: elog.h:39
#define ereport(elevel,...)
Definition: elog.h:149
void err(int eval, const char *fmt,...)
Definition: err.c:43
void fmgr_symbol(Oid functionId, char **mod, char **fn)
Definition: fmgr.c:281
int MyProcPid
Definition: globals.c:44
char pkglib_path[MAXPGPATH]
Definition: globals.c:77
#define funcname
Definition: indent_codes.h:69
static struct @148 value
#define INSTR_TIME_SET_CURRENT(t)
Definition: instr_time.h:122
#define INSTR_TIME_GET_DOUBLE(t)
Definition: instr_time.h:188
#define INSTR_TIME_ACCUM_DIFF(x, y, z)
Definition: instr_time.h:184
void on_proc_exit(pg_on_exit_callback function, Datum arg)
Definition: ipc.c:309
bool proc_exit_inprogress
Definition: ipc.c:40
int i
Definition: isn.c:73
bool jit_dump_bitcode
Definition: jit.c:34
void jit_release_context(JitContext *context)
Definition: jit.c:137
bool jit_profiling_support
Definition: jit.c:36
bool jit_debugging_support
Definition: jit.c:33
#define PGJIT_OPT3
Definition: jit.h:21
#define PGJIT_INLINE
Definition: jit.h:22
Assert(fmt[strlen(fmt) - 1] !='\n')
List * lappend(List *list, void *datum)
Definition: list.c:338
void list_free(List *list)
Definition: list.c:1545
LLVMTypeRef StructFunctionCallInfoData
Definition: llvmjit.c:79
LLVMTypeRef StructMinimalTupleData
Definition: llvmjit.c:72
LLVMTypeRef StructMinimalTupleTableSlot
Definition: llvmjit.c:77
LLVMTypeRef StructPlanState
Definition: llvmjit.c:86
static LLVMContextRef llvm_context
Definition: llvmjit.c:104
static LLVMOrcJITStackRef llvm_opt3_orc
Definition: llvmjit.c:114
LLVMTypeRef StructExprState
Definition: llvmjit.c:82
static const char * llvm_triple
Definition: llvmjit.c:102
LLVMTypeRef StructExprEvalStep
Definition: llvmjit.c:81
LLVMValueRef llvm_pg_func(LLVMModuleRef mod, const char *funcname)
Definition: llvmjit.c:523
static void llvm_recreate_llvm_context(void)
Definition: llvmjit.c:187
struct LLVMJitHandle LLVMJitHandle
LLVMTypeRef TypeParamBool
Definition: llvmjit.c:67
LLVMTypeRef StructMemoryContextData
Definition: llvmjit.c:78
static void llvm_set_target(void)
Definition: llvmjit.c:1127
static LLVMTargetRef llvm_targetref
Definition: llvmjit.c:107
static bool llvm_session_initialized
Definition: llvmjit.c:94
LLVMTypeRef StructAggStatePerGroupData
Definition: llvmjit.c:84
static void llvm_create_types(void)
Definition: llvmjit.c:1146
LLVMTypeRef llvm_pg_var_type(const char *varname)
Definition: llvmjit.c:481
static void ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
Definition: llvmjit.c:148
static void llvm_session_initialize(void)
Definition: llvmjit.c:911
LLVMTypeRef TypeSizeT
Definition: llvmjit.c:66
static size_t llvm_jit_context_in_use_count
Definition: llvmjit.c:98
static const char * llvm_layout
Definition: llvmjit.c:103
LLVMValueRef llvm_function_reference(LLVMJitContext *context, LLVMBuilderRef builder, LLVMModuleRef mod, FunctionCallInfo fcinfo)
Definition: llvmjit.c:600
PG_MODULE_MAGIC
Definition: llvmjit.c:158
char * llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
Definition: llvmjit.c:361
static LLVMOrcJITStackRef llvm_opt0_orc
Definition: llvmjit.c:113
static void ResOwnerReleaseJitContext(Datum res)
Definition: llvmjit.c:1429
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
Definition: llvmjit.c:1200
LLVMTypeRef llvm_pg_var_func_type(const char *varname)
Definition: llvmjit.c:501
LLVMTypeRef StructTupleTableSlot
Definition: llvmjit.c:74
LLVMTypeRef TypeStorageBool
Definition: llvmjit.c:68
static LLVMTypeRef load_return_type(LLVMModuleRef mod, const char *name)
Definition: llvmjit.c:1107
static size_t llvm_generation
Definition: llvmjit.c:95
static void ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
Definition: llvmjit.c:153
static void llvm_compile_module(LLVMJitContext *context)
Definition: llvmjit.c:760
LLVMTypeRef StructHeapTupleTableSlot
Definition: llvmjit.c:76
static LLVMModuleRef llvm_types_module
Definition: llvmjit.c:92
static void llvm_shutdown(int code, Datum arg)
Definition: llvmjit.c:1040
LLVMJitContext * llvm_create_context(int jitFlags)
Definition: llvmjit.c:237
static const ResourceOwnerDesc jit_resowner_desc
Definition: llvmjit.c:137
#define LLVMJIT_LLVM_CONTEXT_REUSE_MAX
Definition: llvmjit.c:50
static void llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
Definition: llvmjit.c:551
LLVMModuleRef llvm_mutable_module(LLVMJitContext *context)
Definition: llvmjit.c:336
LLVMTypeRef StructAggState
Definition: llvmjit.c:83
LLVMTypeRef TypePGFunction
Definition: llvmjit.c:69
LLVMTypeRef StructTupleDescData
Definition: llvmjit.c:73
LLVMValueRef AttributeTemplate
Definition: llvmjit.c:88
static size_t llvm_llvm_context_reuse_count
Definition: llvmjit.c:101
LLVMTypeRef StructExprContext
Definition: llvmjit.c:80
static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
Definition: llvmjit.c:663
static uint64_t llvm_resolve_symbol(const char *name, void *ctx)
Definition: llvmjit.c:1234
LLVMTypeRef StructHeapTupleHeaderData
Definition: llvmjit.c:75
LLVMTypeRef StructHeapTupleData
Definition: llvmjit.c:71
static void llvm_release_context(JitContext *context)
Definition: llvmjit.c:266
LLVMTypeRef StructNullableDatum
Definition: llvmjit.c:70
LLVMValueRef ExecEvalSubroutineTemplate
Definition: llvmjit.c:89
LLVMValueRef ExecEvalBoolSubroutineTemplate
Definition: llvmjit.c:90
void _PG_jit_provider_init(JitProviderCallbacks *cb)
Definition: llvmjit.c:165
void * llvm_get_function(LLVMJitContext *context, const char *funcname)
Definition: llvmjit.c:382
LLVMTypeRef StructAggStatePerTransData
Definition: llvmjit.c:85
void llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
Definition: llvmjit.c:579
void llvm_enter_fatal_on_oom(void)
bool llvm_in_fatal_on_oom(void)
void llvm_assert_in_fatal_section(void)
void llvm_reset_after_error(void)
void llvm_leave_fatal_on_oom(void)
bool llvm_compile_expr(ExprState *state)
Definition: llvmjit_expr.c:78
void llvm_inline(LLVMModuleRef M)
void llvm_inline_reset_caches(void)
LLVMTypeRef LLVMGetFunctionReturnType(LLVMValueRef r)
unsigned LLVMGetAttributeCountAtIndexPG(LLVMValueRef F, uint32 Idx)
LLVMTypeRef LLVMGlobalGetValueType(LLVMValueRef g)
LLVMTypeRef LLVMGetFunctionType(LLVMValueRef r)
static struct pg_tm tm
Definition: localtime.c:104
char * pnstrdup(const char *in, Size len)
Definition: mcxt.c:1655
char * pstrdup(const char *in)
Definition: mcxt.c:1644
void pfree(void *pointer)
Definition: mcxt.c:1456
MemoryContext TopMemoryContext
Definition: mcxt.c:141
void * palloc0(Size size)
Definition: mcxt.c:1257
void * MemoryContextAllocZero(MemoryContext context, Size size)
Definition: mcxt.c:1064
void * MemoryContextAlloc(MemoryContext context, Size size)
Definition: mcxt.c:1021
void * palloc(Size size)
Definition: mcxt.c:1226
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:138
void * arg
#define MAXPGPATH
static char * filename
Definition: pg_dumpall.c:121
#define lfirst(lc)
Definition: pg_list.h:172
#define NIL
Definition: pg_list.h:68
static char ** options
static char * buf
Definition: pg_test_fsync.c:73
#define snprintf
Definition: port.h:238
static Datum PointerGetDatum(const void *X)
Definition: postgres.h:322
uintptr_t Datum
Definition: postgres.h:64
static Pointer DatumGetPointer(Datum X)
Definition: postgres.h:312
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
ResourceOwner CurrentResourceOwner
Definition: resowner.c:164
void ResourceOwnerForget(ResourceOwner owner, Datum value, const ResourceOwnerDesc *kind)
Definition: resowner.c:555
void ResourceOwnerRemember(ResourceOwner owner, Datum value, const ResourceOwnerDesc *kind)
Definition: resowner.c:520
void ResourceOwnerEnlarge(ResourceOwner owner)
Definition: resowner.c:448
#define RELEASE_PRIO_JIT_CONTEXTS
Definition: resowner.h:66
@ RESOURCE_RELEASE_BEFORE_LOCKS
Definition: resowner.h:54
static void error(void)
Definition: sql-dyntest.c:147
PGFunction fn_addr
Definition: fmgr.h:58
Oid fn_oid
Definition: fmgr.h:59
FmgrInfo * flinfo
Definition: fmgr.h:87
Definition: jit.h:58
ResourceOwner resowner
Definition: jit.h:62
JitProviderCompileExprCB compile_expr
Definition: jit.h:80
JitProviderResetAfterErrorCB reset_after_error
Definition: jit.h:78
JitProviderReleaseContextCB release_context
Definition: jit.h:79
LLVMOrcJITStackRef stack
Definition: llvmjit.c:59
LLVMOrcModuleHandle orc_handle
Definition: llvmjit.c:60
const char * name
Definition: resowner.h:93
Definition: type.h:95
Definition: c.h:730
const char * name