PostgreSQL Source Code  git master
llvmjit.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * llvmjit.c
4  * Core part of the LLVM JIT provider.
5  *
6  * Copyright (c) 2016-2020, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/jit/llvm/llvmjit.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres.h"
15 
16 #include <llvm-c/Analysis.h>
17 #include <llvm-c/BitReader.h>
18 #include <llvm-c/BitWriter.h>
19 #include <llvm-c/Core.h>
20 #include <llvm-c/ExecutionEngine.h>
21 #include <llvm-c/OrcBindings.h>
22 #include <llvm-c/Support.h>
23 #include <llvm-c/Target.h>
24 #include <llvm-c/Transforms/IPO.h>
25 #include <llvm-c/Transforms/PassManagerBuilder.h>
26 #include <llvm-c/Transforms/Scalar.h>
27 #if LLVM_VERSION_MAJOR > 6
28 #include <llvm-c/Transforms/Utils.h>
29 #endif
30 
31 #include "jit/llvmjit.h"
32 #include "jit/llvmjit_emit.h"
33 #include "miscadmin.h"
34 #include "portability/instr_time.h"
35 #include "storage/ipc.h"
36 #include "utils/memutils.h"
37 #include "utils/resowner_private.h"
38 
39 /* Handle of a module emitted via ORC JIT */
40 typedef struct LLVMJitHandle
41 {
42  LLVMOrcJITStackRef stack;
43  LLVMOrcModuleHandle orc_handle;
45 
46 
47 /* types & functions commonly needed for JITing */
48 LLVMTypeRef TypeSizeT;
49 LLVMTypeRef TypeParamBool;
50 LLVMTypeRef TypeStorageBool;
51 LLVMTypeRef TypePGFunction;
52 LLVMTypeRef StructNullableDatum;
57 LLVMTypeRef StructHeapTupleData;
60 LLVMTypeRef StructBlockId;
62 LLVMTypeRef StructTupleConstr;
63 LLVMTypeRef StructTupleDescData;
68 LLVMTypeRef StructPGFinfoRecord;
69 LLVMTypeRef StructFmgrInfo;
71 LLVMTypeRef StructExprContext;
72 LLVMTypeRef StructExprEvalStep;
73 LLVMTypeRef StructExprState;
74 LLVMTypeRef StructAggState;
77 
78 LLVMValueRef AttributeTemplate;
79 LLVMValueRef FuncStrlen;
80 LLVMValueRef FuncVarsizeAny;
85 LLVMValueRef FuncExecEvalSysVar;
87 LLVMValueRef FuncExecAggInitGroup;
88 
89 
90 static bool llvm_session_initialized = false;
91 static size_t llvm_generation = 0;
92 static const char *llvm_triple = NULL;
93 static const char *llvm_layout = NULL;
94 
95 
96 static LLVMTargetMachineRef llvm_opt0_targetmachine;
97 static LLVMTargetMachineRef llvm_opt3_targetmachine;
98 
99 static LLVMTargetRef llvm_targetref;
100 static LLVMOrcJITStackRef llvm_opt0_orc;
101 static LLVMOrcJITStackRef llvm_opt3_orc;
102 
103 
104 static void llvm_release_context(JitContext *context);
105 static void llvm_session_initialize(void);
106 static void llvm_shutdown(int code, Datum arg);
107 static void llvm_compile_module(LLVMJitContext *context);
108 static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
109 
110 static void llvm_create_types(void);
111 static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
112 
113 
115 
116 
117 /*
118  * Initialize LLVM JIT provider.
119  */
120 void
122 {
126 }
127 
128 /*
129  * Create a context for JITing work.
130  *
131  * The context, including subsidiary resources, will be cleaned up either when
132  * the context is explicitly released, or when the lifetime of
133  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
134  */
135 LLVMJitContext *
136 llvm_create_context(int jitFlags)
137 {
138  LLVMJitContext *context;
139 
141 
143 
145 
147  sizeof(LLVMJitContext));
148  context->base.flags = jitFlags;
149 
150  /* ensure cleanup */
151  context->base.resowner = CurrentResourceOwner;
153 
154  return context;
155 }
156 
157 /*
158  * Release resources required by one llvm context.
159  */
160 static void
162 {
163  LLVMJitContext *llvm_context = (LLVMJitContext *) context;
164 
166 
167  /*
168  * When this backend is exiting, don't clean up LLVM. As an error might
169  * have occurred from within LLVM, we do not want to risk reentering. All
170  * resource cleanup is going to happen through process exit.
171  */
173  {
174  if (llvm_context->module)
175  {
176  LLVMDisposeModule(llvm_context->module);
177  llvm_context->module = NULL;
178  }
179 
180  while (llvm_context->handles != NIL)
181  {
182  LLVMJitHandle *jit_handle;
183 
184  jit_handle = (LLVMJitHandle *) linitial(llvm_context->handles);
185  llvm_context->handles = list_delete_first(llvm_context->handles);
186 
187  LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
188  pfree(jit_handle);
189  }
190  }
191 }
192 
193 /*
194  * Return module which may be modified, e.g. by creating new functions.
195  */
196 LLVMModuleRef
197 llvm_mutable_module(LLVMJitContext *context)
198 {
200 
201  /*
202  * If there's no in-progress module, create a new one.
203  */
204  if (!context->module)
205  {
206  context->compiled = false;
207  context->module_generation = llvm_generation++;
208  context->module = LLVMModuleCreateWithName("pg");
209  LLVMSetTarget(context->module, llvm_triple);
210  LLVMSetDataLayout(context->module, llvm_layout);
211  }
212 
213  return context->module;
214 }
215 
216 /*
217  * Expand function name to be non-conflicting. This should be used by code
218  * generating code, when adding new externally visible function definitions to
219  * a Module.
220  */
221 char *
222 llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
223 {
224  Assert(context->module != NULL);
225 
226  context->base.instr.created_functions++;
227 
228  /*
229  * Previously we used dots to separate, but turns out some tools, e.g.
230  * GDB, don't like that and truncate name.
231  */
232  return psprintf("%s_%zu_%d",
233  basename,
234  context->module_generation,
235  context->counter++);
236 }
237 
238 /*
239  * Return pointer to function funcname, which has to exist. If there's pending
240  * code to be optimized and emitted, do so first.
241  */
242 void *
243 llvm_get_function(LLVMJitContext *context, const char *funcname)
244 {
245  LLVMOrcTargetAddress addr = 0;
246 #if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
247  ListCell *lc;
248 #endif
249 
251 
252  /*
253  * If there is a pending / not emitted module, compile and emit now.
254  * Otherwise we might not find the [correct] function.
255  */
256  if (!context->compiled)
257  {
258  llvm_compile_module(context);
259  }
260 
261  /*
262  * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
263  * to mangle here.
264  */
265 
266 #if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
267  foreach(lc, context->handles)
268  {
269  LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
270 
271  addr = 0;
272  if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
273  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
274  if (addr)
275  return (void *) (uintptr_t) addr;
276  }
277 
278 #else
279 
280 #if LLVM_VERSION_MAJOR < 5
281  if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
282  return (void *) (uintptr_t) addr;
283  if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
284  return (void *) (uintptr_t) addr;
285 #else
286  if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
287  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
288  if (addr)
289  return (void *) (uintptr_t) addr;
290  if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
291  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
292  if (addr)
293  return (void *) (uintptr_t) addr;
294 #endif /* LLVM_VERSION_MAJOR */
295 
296 #endif /* HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN */
297 
298  elog(ERROR, "failed to JIT: %s", funcname);
299 
300  return NULL;
301 }
302 
303 /*
304  * Return declaration for passed function, adding it to the module if
305  * necessary.
306  *
307  * This is used to make functions imported by llvm_create_types() known to the
308  * module that's currently being worked on.
309  */
310 LLVMValueRef
311 llvm_get_decl(LLVMModuleRef mod, LLVMValueRef v_src)
312 {
313  LLVMValueRef v_fn;
314 
315  /* don't repeatedly add function */
316  v_fn = LLVMGetNamedFunction(mod, LLVMGetValueName(v_src));
317  if (v_fn)
318  return v_fn;
319 
320  v_fn = LLVMAddFunction(mod,
321  LLVMGetValueName(v_src),
322  LLVMGetElementType(LLVMTypeOf(v_src)));
323  llvm_copy_attributes(v_src, v_fn);
324 
325  return v_fn;
326 }
327 
328 /*
329  * Copy attributes from one function to another.
330  */
331 void
332 llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
333 {
334  int num_attributes;
335  int attno;
336  LLVMAttributeRef *attrs;
337 
338  num_attributes =
339  LLVMGetAttributeCountAtIndex(v_from, LLVMAttributeFunctionIndex);
340 
341  attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
342  LLVMGetAttributesAtIndex(v_from, LLVMAttributeFunctionIndex, attrs);
343 
344  for (attno = 0; attno < num_attributes; attno++)
345  {
346  LLVMAddAttributeAtIndex(v_to, LLVMAttributeFunctionIndex,
347  attrs[attno]);
348  }
349 }
350 
351 /*
352  * Return a callable LLVMValueRef for fcinfo.
353  */
354 LLVMValueRef
355 llvm_function_reference(LLVMJitContext *context,
356  LLVMBuilderRef builder,
357  LLVMModuleRef mod,
358  FunctionCallInfo fcinfo)
359 {
360  char *modname;
361  char *basename;
362  char *funcname;
363 
364  LLVMValueRef v_fn;
365 
366  fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
367 
368  if (modname != NULL && basename != NULL)
369  {
370  /* external function in loadable library */
371  funcname = psprintf("pgextern.%s.%s", modname, basename);
372  }
373  else if (basename != NULL)
374  {
375  /* internal function */
376  funcname = psprintf("%s", basename);
377  }
378  else
379  {
380  /*
381  * Function we don't know to handle, return pointer. We do so by
382  * creating a global constant containing a pointer to the function.
383  * Makes IR more readable.
384  */
385  LLVMValueRef v_fn_addr;
386 
387  funcname = psprintf("pgoidextern.%u",
388  fcinfo->flinfo->fn_oid);
389  v_fn = LLVMGetNamedGlobal(mod, funcname);
390  if (v_fn != 0)
391  return LLVMBuildLoad(builder, v_fn, "");
392 
393  v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
394 
395  v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
396  LLVMSetInitializer(v_fn, v_fn_addr);
397  LLVMSetGlobalConstant(v_fn, true);
398 
399  return LLVMBuildLoad(builder, v_fn, "");
400  }
401 
402  /* check if function already has been added */
403  v_fn = LLVMGetNamedFunction(mod, funcname);
404  if (v_fn != 0)
405  return v_fn;
406 
407  v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction));
408 
409  return v_fn;
410 }
411 
412 /*
413  * Optimize code in module using the flags set in context.
414  */
415 static void
416 llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
417 {
418  LLVMPassManagerBuilderRef llvm_pmb;
419  LLVMPassManagerRef llvm_mpm;
420  LLVMPassManagerRef llvm_fpm;
421  LLVMValueRef func;
422  int compile_optlevel;
423 
424  if (context->base.flags & PGJIT_OPT3)
425  compile_optlevel = 3;
426  else
427  compile_optlevel = 0;
428 
429  /*
430  * Have to create a new pass manager builder every pass through, as the
431  * inliner has some per-builder state. Otherwise one ends up only inlining
432  * a function the first time though.
433  */
434  llvm_pmb = LLVMPassManagerBuilderCreate();
435  LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
436  llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
437 
438  if (context->base.flags & PGJIT_OPT3)
439  {
440  /* TODO: Unscientifically determined threshold */
441  LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
442  }
443  else
444  {
445  /* we rely on mem2reg heavily, so emit even in the O0 case */
446  LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
447  }
448 
449  LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
450 
451  /*
452  * Do function level optimization. This could be moved to the point where
453  * functions are emitted, to reduce memory usage a bit.
454  */
455  LLVMInitializeFunctionPassManager(llvm_fpm);
456  for (func = LLVMGetFirstFunction(context->module);
457  func != NULL;
458  func = LLVMGetNextFunction(func))
459  LLVMRunFunctionPassManager(llvm_fpm, func);
460  LLVMFinalizeFunctionPassManager(llvm_fpm);
461  LLVMDisposePassManager(llvm_fpm);
462 
463  /*
464  * Perform module level optimization. We do so even in the non-optimized
465  * case, so always-inline functions etc get inlined. It's cheap enough.
466  */
467  llvm_mpm = LLVMCreatePassManager();
468  LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
469  llvm_mpm);
470  /* always use always-inliner pass */
471  if (!(context->base.flags & PGJIT_OPT3))
472  LLVMAddAlwaysInlinerPass(llvm_mpm);
473  /* if doing inlining, but no expensive optimization, add inlining pass */
474  if (context->base.flags & PGJIT_INLINE
475  && !(context->base.flags & PGJIT_OPT3))
476  LLVMAddFunctionInliningPass(llvm_mpm);
477  LLVMRunPassManager(llvm_mpm, context->module);
478  LLVMDisposePassManager(llvm_mpm);
479 
480  LLVMPassManagerBuilderDispose(llvm_pmb);
481 }
482 
483 /*
484  * Emit code for the currently pending module.
485  */
486 static void
487 llvm_compile_module(LLVMJitContext *context)
488 {
489  LLVMOrcModuleHandle orc_handle;
490  MemoryContext oldcontext;
491  static LLVMOrcJITStackRef compile_orc;
492  instr_time starttime;
493  instr_time endtime;
494 
495  if (context->base.flags & PGJIT_OPT3)
496  compile_orc = llvm_opt3_orc;
497  else
498  compile_orc = llvm_opt0_orc;
499 
500  /* perform inlining */
501  if (context->base.flags & PGJIT_INLINE)
502  {
503  INSTR_TIME_SET_CURRENT(starttime);
504  llvm_inline(context->module);
505  INSTR_TIME_SET_CURRENT(endtime);
506  INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
507  endtime, starttime);
508  }
509 
510  if (jit_dump_bitcode)
511  {
512  char *filename;
513 
514  filename = psprintf("%u.%zu.bc",
515  MyProcPid,
516  context->module_generation);
517  LLVMWriteBitcodeToFile(context->module, filename);
518  pfree(filename);
519  }
520 
521 
522  /* optimize according to the chosen optimization settings */
523  INSTR_TIME_SET_CURRENT(starttime);
524  llvm_optimize_module(context, context->module);
525  INSTR_TIME_SET_CURRENT(endtime);
526  INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
527  endtime, starttime);
528 
529  if (jit_dump_bitcode)
530  {
531  char *filename;
532 
533  filename = psprintf("%u.%zu.optimized.bc",
534  MyProcPid,
535  context->module_generation);
536  LLVMWriteBitcodeToFile(context->module, filename);
537  pfree(filename);
538  }
539 
540  /*
541  * Emit the code. Note that this can, depending on the optimization
542  * settings, take noticeable resources as code emission executes low-level
543  * instruction combining/selection passes etc. Without optimization a
544  * faster instruction selection mechanism is used.
545  */
546  INSTR_TIME_SET_CURRENT(starttime);
547 #if LLVM_VERSION_MAJOR > 6
548  {
549  if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, context->module,
550  llvm_resolve_symbol, NULL))
551  {
552  elog(ERROR, "failed to JIT module");
553  }
554 
555  /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
556  }
557 #elif LLVM_VERSION_MAJOR > 4
558  {
559  LLVMSharedModuleRef smod;
560 
561  smod = LLVMOrcMakeSharedModule(context->module);
562  if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, smod,
563  llvm_resolve_symbol, NULL))
564  {
565  elog(ERROR, "failed to JIT module");
566  }
567  LLVMOrcDisposeSharedModuleRef(smod);
568  }
569 #else /* LLVM 4.0 and 3.9 */
570  {
571  orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
572  llvm_resolve_symbol, NULL);
573  LLVMDisposeModule(context->module);
574  }
575 #endif
576  INSTR_TIME_SET_CURRENT(endtime);
577  INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
578  endtime, starttime);
579 
580  context->module = NULL;
581  context->compiled = true;
582 
583  /* remember emitted code for cleanup and lookups */
585  {
586  LLVMJitHandle *handle;
587 
588  handle = (LLVMJitHandle *) palloc(sizeof(LLVMJitHandle));
589  handle->stack = compile_orc;
590  handle->orc_handle = orc_handle;
591 
592  context->handles = lappend(context->handles, handle);
593  }
594  MemoryContextSwitchTo(oldcontext);
595 
596  ereport(DEBUG1,
597  (errmsg("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
598  INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
599  INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
600  INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
601  errhidestmt(true),
602  errhidecontext(true)));
603 }
604 
605 /*
606  * Per session initialization.
607  */
608 static void
610 {
611  MemoryContext oldcontext;
612  char *error = NULL;
613  char *cpu = NULL;
614  char *features = NULL;
615 
617  return;
618 
620 
621  LLVMInitializeNativeTarget();
622  LLVMInitializeNativeAsmPrinter();
623  LLVMInitializeNativeAsmParser();
624 
625  /*
626  * Synchronize types early, as that also includes inferring the target
627  * triple.
628  */
630 
631  if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
632  {
633  elog(FATAL, "failed to query triple %s\n", error);
634  }
635 
636  /*
637  * We want the generated code to use all available features. Therefore
638  * grab the host CPU string and detect features of the current CPU. The
639  * latter is needed because some CPU architectures default to enabling
640  * features not all CPUs have (weird, huh).
641  */
642  cpu = LLVMGetHostCPUName();
643  features = LLVMGetHostCPUFeatures();
644  elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
645  cpu, features);
646 
648  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
649  LLVMCodeGenLevelNone,
650  LLVMRelocDefault,
651  LLVMCodeModelJITDefault);
653  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
654  LLVMCodeGenLevelAggressive,
655  LLVMRelocDefault,
656  LLVMCodeModelJITDefault);
657 
658  LLVMDisposeMessage(cpu);
659  cpu = NULL;
660  LLVMDisposeMessage(features);
661  features = NULL;
662 
663  /* force symbols in main binary to be loaded */
664  LLVMLoadLibraryPermanently(NULL);
665 
666  llvm_opt0_orc = LLVMOrcCreateInstance(llvm_opt0_targetmachine);
667  llvm_opt3_orc = LLVMOrcCreateInstance(llvm_opt3_targetmachine);
668 
669 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
671  {
672  LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
673 
674  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
675  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
676  }
677 #endif
678 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
680  {
681  LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
682 
683  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
684  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
685  }
686 #endif
687 
689 
691 
692  MemoryContextSwitchTo(oldcontext);
693 }
694 
695 static void
697 {
698  /* unregister profiling support, needs to be flushed to be useful */
699 
700  if (llvm_opt3_orc)
701  {
702 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
704  LLVMOrcUnregisterPerf(llvm_opt3_orc);
705 #endif
706  LLVMOrcDisposeInstance(llvm_opt3_orc);
707  llvm_opt3_orc = NULL;
708  }
709 
710  if (llvm_opt0_orc)
711  {
712 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
714  LLVMOrcUnregisterPerf(llvm_opt0_orc);
715 #endif
716  LLVMOrcDisposeInstance(llvm_opt0_orc);
717  llvm_opt0_orc = NULL;
718  }
719 }
720 
721 /* helper for llvm_create_types, returning a global var's type */
722 static LLVMTypeRef
723 load_type(LLVMModuleRef mod, const char *name)
724 {
725  LLVMValueRef value;
726  LLVMTypeRef typ;
727 
728  /* this'll return a *pointer* to the global */
729  value = LLVMGetNamedGlobal(mod, name);
730  if (!value)
731  elog(ERROR, "type %s is unknown", name);
732 
733  /* therefore look at the contained type and return that */
734  typ = LLVMTypeOf(value);
735  Assert(typ != NULL);
736  typ = LLVMGetElementType(typ);
737  Assert(typ != NULL);
738  return typ;
739 }
740 
741 /* helper for llvm_create_types, returning a function's return type */
742 static LLVMTypeRef
743 load_return_type(LLVMModuleRef mod, const char *name)
744 {
745  LLVMValueRef value;
746  LLVMTypeRef typ;
747 
748  /* this'll return a *pointer* to the function */
749  value = LLVMGetNamedFunction(mod, name);
750  if (!value)
751  elog(ERROR, "function %s is unknown", name);
752 
753  /* get type of function pointer */
754  typ = LLVMTypeOf(value);
755  Assert(typ != NULL);
756  /* dereference pointer */
757  typ = LLVMGetElementType(typ);
758  Assert(typ != NULL);
759  /* and look at return type */
760  typ = LLVMGetReturnType(typ);
761  Assert(typ != NULL);
762 
763  return typ;
764 }
765 
766 /*
767  * Load required information, types, function signatures from llvmjit_types.c
768  * and make them available in global variables.
769  *
770  * Those global variables are then used while emitting code.
771  */
772 static void
774 {
775  char path[MAXPGPATH];
776  LLVMMemoryBufferRef buf;
777  char *msg;
778  LLVMModuleRef mod = NULL;
779 
780  snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
781 
782  /* open file */
783  if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
784  {
785  elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
786  path, msg);
787  }
788 
789  /* eagerly load contents, going to need it all */
790  if (LLVMParseBitcode2(buf, &mod))
791  {
792  elog(ERROR, "LLVMParseBitcode2 of %s failed", path);
793  }
794  LLVMDisposeMemoryBuffer(buf);
795 
796  /*
797  * Load triple & layout from clang emitted file so we're guaranteed to be
798  * compatible.
799  */
800  llvm_triple = pstrdup(LLVMGetTarget(mod));
801  llvm_layout = pstrdup(LLVMGetDataLayoutStr(mod));
802 
803  TypeSizeT = load_type(mod, "TypeSizeT");
804  TypeParamBool = load_return_type(mod, "FunctionReturningBool");
805  TypeStorageBool = load_type(mod, "TypeStorageBool");
806  TypePGFunction = load_type(mod, "TypePGFunction");
807  StructNullableDatum = load_type(mod, "StructNullableDatum");
808  StructExprContext = load_type(mod, "StructExprContext");
809  StructExprEvalStep = load_type(mod, "StructExprEvalStep");
810  StructExprState = load_type(mod, "StructExprState");
811  StructFunctionCallInfoData = load_type(mod, "StructFunctionCallInfoData");
812  StructMemoryContextData = load_type(mod, "StructMemoryContextData");
813  StructTupleTableSlot = load_type(mod, "StructTupleTableSlot");
814  StructHeapTupleTableSlot = load_type(mod, "StructHeapTupleTableSlot");
815  StructMinimalTupleTableSlot = load_type(mod, "StructMinimalTupleTableSlot");
816  StructHeapTupleData = load_type(mod, "StructHeapTupleData");
817  StructTupleDescData = load_type(mod, "StructTupleDescData");
818  StructAggState = load_type(mod, "StructAggState");
819  StructAggStatePerGroupData = load_type(mod, "StructAggStatePerGroupData");
820  StructAggStatePerTransData = load_type(mod, "StructAggStatePerTransData");
821 
822  AttributeTemplate = LLVMGetNamedFunction(mod, "AttributeTemplate");
823  FuncStrlen = LLVMGetNamedFunction(mod, "strlen");
824  FuncVarsizeAny = LLVMGetNamedFunction(mod, "varsize_any");
825  FuncSlotGetsomeattrsInt = LLVMGetNamedFunction(mod, "slot_getsomeattrs_int");
826  FuncSlotGetmissingattrs = LLVMGetNamedFunction(mod, "slot_getmissingattrs");
827  FuncMakeExpandedObjectReadOnlyInternal = LLVMGetNamedFunction(mod, "MakeExpandedObjectReadOnlyInternal");
828  FuncExecEvalSubscriptingRef = LLVMGetNamedFunction(mod, "ExecEvalSubscriptingRef");
829  FuncExecEvalSysVar = LLVMGetNamedFunction(mod, "ExecEvalSysVar");
830  FuncExecAggTransReparent = LLVMGetNamedFunction(mod, "ExecAggTransReparent");
831  FuncExecAggInitGroup = LLVMGetNamedFunction(mod, "ExecAggInitGroup");
832 
833  /*
834  * Leave the module alive, otherwise references to function would be
835  * dangling.
836  */
837 }
838 
839 /*
840  * Split a symbol into module / function parts. If the function is in the
841  * main binary (or an external library) *modname will be NULL.
842  */
843 void
844 llvm_split_symbol_name(const char *name, char **modname, char **funcname)
845 {
846  *modname = NULL;
847  *funcname = NULL;
848 
849  /*
850  * Module function names are pgextern.$module.$funcname
851  */
852  if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
853  {
854  /*
855  * Symbol names cannot contain a ., therefore we can split based on
856  * first and last occurrence of one.
857  */
858  *funcname = rindex(name, '.');
859  (*funcname)++; /* jump over . */
860 
861  *modname = pnstrdup(name + strlen("pgextern."),
862  *funcname - name - strlen("pgextern.") - 1);
863  Assert(funcname);
864 
865  *funcname = pstrdup(*funcname);
866  }
867  else
868  {
869  *modname = NULL;
870  *funcname = pstrdup(name);
871  }
872 }
873 
874 /*
875  * Attempt to resolve symbol, so LLVM can emit a reference to it.
876  */
877 static uint64_t
878 llvm_resolve_symbol(const char *symname, void *ctx)
879 {
880  uintptr_t addr;
881  char *funcname;
882  char *modname;
883 
884  /*
885  * macOS prefixes all object level symbols with an underscore. But neither
886  * dlsym() nor PG's inliner expect that. So undo.
887  */
888 #if defined(__darwin__)
889  if (symname[0] != '_')
890  elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
891  symname++;
892 #endif
893 
894  llvm_split_symbol_name(symname, &modname, &funcname);
895 
896  /* functions that aren't resolved to names shouldn't ever get here */
897  Assert(funcname);
898 
899  if (modname)
900  addr = (uintptr_t) load_external_function(modname, funcname,
901  true, NULL);
902  else
903  addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
904 
905  pfree(funcname);
906  if (modname)
907  pfree(modname);
908 
909  /* let LLVM will error out - should never happen */
910  if (!addr)
911  elog(WARNING, "failed to resolve name %s", symname);
912 
913  return (uint64_t) addr;
914 }
LLVMTypeRef StructPGFinfoRecord
Definition: llvmjit.c:68
#define NIL
Definition: pg_list.h:65
JitProviderResetAfterErrorCB reset_after_error
Definition: jit.h:75
LLVMTypeRef StructTupleConstr
Definition: llvmjit.c:62
#define DEBUG1
Definition: elog.h:25
LLVMTypeRef StructFunctionCallInfoData
Definition: llvmjit.c:70
int MyProcPid
Definition: globals.c:40
static void llvm_compile_module(LLVMJitContext *context)
Definition: llvmjit.c:487
char * pnstrdup(const char *in, Size len)
Definition: mcxt.c:1197
PG_MODULE_MAGIC
Definition: llvmjit.c:114
static LLVMTypeRef load_type(LLVMModuleRef mod, const char *name)
Definition: llvmjit.c:723
static void error(void)
Definition: sql-dyntest.c:147
LLVMTypeRef StructHeapTupleFieldsField3
Definition: llvmjit.c:53
LLVMTypeRef StructMinimalTupleData
Definition: llvmjit.c:58
LLVMOrcJITStackRef stack
Definition: llvmjit.c:42
PGFunction fn_addr
Definition: fmgr.h:58
LLVMTypeRef StructExprEvalStep
Definition: llvmjit.c:72
#define PointerGetDatum(X)
Definition: postgres.h:556
LLVMTypeRef TypeSizeT
Definition: llvmjit.c:48
LLVMTypeRef StructBlockId
Definition: llvmjit.c:60
LLVMTypeRef StructExprState
Definition: llvmjit.c:73
LLVMTypeRef StructHeapTupleFields
Definition: llvmjit.c:54
ResourceOwner CurrentResourceOwner
Definition: resowner.c:142
static LLVMTargetMachineRef llvm_opt3_targetmachine
Definition: llvmjit.c:97
char * pstrdup(const char *in)
Definition: mcxt.c:1186
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
bool jit_debugging_support
Definition: jit.c:34
LLVMValueRef FuncVarsizeAny
Definition: llvmjit.c:80
struct timeval instr_time
Definition: instr_time.h:150
static void llvm_shutdown(int code, Datum arg)
Definition: llvmjit.c:696
LLVMTypeRef StructNullableDatum
Definition: llvmjit.c:52
LLVMTypeRef StructItemPointerData
Definition: llvmjit.c:59
LLVMValueRef llvm_get_decl(LLVMModuleRef mod, LLVMValueRef v_src)
Definition: llvmjit.c:311
Definition: jit.h:54
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
void _PG_jit_provider_init(JitProviderCallbacks *cb)
Definition: llvmjit.c:121
static struct @145 value
LLVMValueRef llvm_function_reference(LLVMJitContext *context, LLVMBuilderRef builder, LLVMModuleRef mod, FunctionCallInfo fcinfo)
Definition: llvmjit.c:355
int errhidestmt(bool hide_stmt)
Definition: elog.c:1150
#define INSTR_TIME_ACCUM_DIFF(x, y, z)
Definition: instr_time.h:182
static const char * llvm_layout
Definition: llvmjit.c:93
LLVMJitContext * llvm_create_context(int jitFlags)
Definition: llvmjit.c:136
LLVMTypeRef StructFmgrInfo
Definition: llvmjit.c:69
#define INSTR_TIME_GET_DOUBLE(t)
Definition: instr_time.h:199
LLVMTypeRef StructHeapTupleTableSlot
Definition: llvmjit.c:65
LLVMTypeRef StructAggState
Definition: llvmjit.c:74
static LLVMTypeRef load_return_type(LLVMModuleRef mod, const char *name)
Definition: llvmjit.c:743
void fmgr_symbol(Oid functionId, char **mod, char **fn)
Definition: fmgr.c:281
static const char * llvm_triple
Definition: llvmjit.c:92
LLVMTypeRef StructHeapTupleData
Definition: llvmjit.c:57
LLVMTypeRef StructHeapTupleDataChoice
Definition: llvmjit.c:56
#define PGJIT_OPT3
Definition: jit.h:21
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
Definition: llvmjit.c:844
void pfree(void *pointer)
Definition: mcxt.c:1056
#define linitial(l)
Definition: pg_list.h:195
#define ERROR
Definition: elog.h:43
struct LLVMJitHandle LLVMJitHandle
#define FATAL
Definition: elog.h:52
static uint64_t llvm_resolve_symbol(const char *name, void *ctx)
Definition: llvmjit.c:878
#define MAXPGPATH
LLVMValueRef FuncExecAggInitGroup
Definition: llvmjit.c:87
LLVMOrcModuleHandle orc_handle
Definition: llvmjit.c:43
char * llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
Definition: llvmjit.c:222
#define DEBUG2
Definition: elog.h:24
void llvm_assert_in_fatal_section(void)
static char * buf
Definition: pg_test_fsync.c:67
LLVMTypeRef TypeStorageBool
Definition: llvmjit.c:50
LLVMValueRef FuncExecEvalSubscriptingRef
Definition: llvmjit.c:84
LLVMValueRef FuncMakeExpandedObjectReadOnlyInternal
Definition: llvmjit.c:83
void before_shmem_exit(pg_on_exit_callback function, Datum arg)
Definition: ipc.c:333
LLVMValueRef FuncSlotGetsomeattrsInt
Definition: llvmjit.c:81
LLVMTypeRef StructMemoryContextData
Definition: llvmjit.c:67
PGFunction load_external_function(const char *filename, const char *funcname, bool signalNotFound, void **filehandle)
Definition: dfmgr.c:107
LLVMTypeRef StructExprContext
Definition: llvmjit.c:71
#define ereport(elevel, rest)
Definition: elog.h:141
MemoryContext TopMemoryContext
Definition: mcxt.c:44
static LLVMOrcJITStackRef llvm_opt3_orc
Definition: llvmjit.c:101
List * lappend(List *list, void *datum)
Definition: list.c:322
LLVMTypeRef TypePGFunction
Definition: llvmjit.c:51
#define WARNING
Definition: elog.h:40
LLVMTypeRef StructAggStatePerTransData
Definition: llvmjit.c:76
JitProviderCompileExprCB compile_expr
Definition: jit.h:77
void ResourceOwnerRememberJIT(ResourceOwner owner, Datum handle)
Definition: resowner.c:1334
uintptr_t Datum
Definition: postgres.h:367
FmgrInfo * flinfo
Definition: fmgr.h:87
#define PGJIT_INLINE
Definition: jit.h:22
LLVMValueRef FuncSlotGetmissingattrs
Definition: llvmjit.c:82
static bool llvm_session_initialized
Definition: llvmjit.c:90
void llvm_reset_after_error(void)
LLVMValueRef FuncStrlen
Definition: llvmjit.c:79
LLVMTypeRef StructMinimalTupleTableSlot
Definition: llvmjit.c:66
static size_t llvm_generation
Definition: llvmjit.c:91
void * MemoryContextAllocZero(MemoryContext context, Size size)
Definition: mcxt.c:839
LLVMTypeRef StructTupleTableSlot
Definition: llvmjit.c:64
LLVMValueRef FuncExecEvalSysVar
Definition: llvmjit.c:85
Oid fn_oid
Definition: fmgr.h:59
LLVMModuleRef llvm_mutable_module(LLVMJitContext *context)
Definition: llvmjit.c:197
static LLVMTargetRef llvm_targetref
Definition: llvmjit.c:99
static void llvm_session_initialize(void)
Definition: llvmjit.c:609
static void llvm_release_context(JitContext *context)
Definition: llvmjit.c:161
#define Assert(condition)
Definition: c.h:739
#define lfirst(lc)
Definition: pg_list.h:190
bool proc_exit_inprogress
Definition: ipc.c:40
bool llvm_compile_expr(ExprState *state)
Definition: llvmjit_expr.c:71
LLVMTypeRef StructTupleDescData
Definition: llvmjit.c:63
LLVMTypeRef TypeParamBool
Definition: llvmjit.c:49
bool jit_profiling_support
Definition: jit.c:37
void * llvm_get_function(LLVMJitContext *context, const char *funcname)
Definition: llvmjit.c:243
bool jit_dump_bitcode
Definition: jit.c:35
#define INSTR_TIME_SET_CURRENT(t)
Definition: instr_time.h:156
const char * name
Definition: encode.c:521
static char * filename
Definition: pg_dumpall.c:90
void llvm_inline(LLVMModuleRef M)
static void llvm_create_types(void)
Definition: llvmjit.c:773
void * palloc(Size size)
Definition: mcxt.c:949
int errmsg(const char *fmt,...)
Definition: elog.c:822
static LLVMTargetMachineRef llvm_opt0_targetmachine
Definition: llvmjit.c:96
void llvm_enter_fatal_on_oom(void)
#define elog(elevel,...)
Definition: elog.h:228
LLVMValueRef AttributeTemplate
Definition: llvmjit.c:78
void * arg
LLVMTypeRef StructHeapTupleHeaderData
Definition: llvmjit.c:55
void ResourceOwnerEnlargeJIT(ResourceOwner owner)
Definition: resowner.c:1323
void llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
Definition: llvmjit.c:332
static LLVMOrcJITStackRef llvm_opt0_orc
Definition: llvmjit.c:100
JitProviderReleaseContextCB release_context
Definition: jit.h:76
#define snprintf
Definition: port.h:192
static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
Definition: llvmjit.c:416
char pkglib_path[MAXPGPATH]
Definition: globals.c:73
LLVMValueRef FuncExecAggTransReparent
Definition: llvmjit.c:86
int errhidecontext(bool hide_ctx)
Definition: elog.c:1169
List * list_delete_first(List *list)
Definition: list.c:861
LLVMTypeRef StructFormPgAttribute
Definition: llvmjit.c:61
LLVMTypeRef StructAggStatePerGroupData
Definition: llvmjit.c:75