PostgreSQL Source Code  git master
llvmjit.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * llvmjit.c
4  * Core part of the LLVM JIT provider.
5  *
6  * Copyright (c) 2016-2020, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/jit/llvm/llvmjit.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres.h"
15 
16 #include <llvm-c/Analysis.h>
17 #include <llvm-c/BitReader.h>
18 #include <llvm-c/BitWriter.h>
19 #include <llvm-c/Core.h>
20 #include <llvm-c/ExecutionEngine.h>
21 #include <llvm-c/OrcBindings.h>
22 #include <llvm-c/Support.h>
23 #include <llvm-c/Target.h>
24 #include <llvm-c/Transforms/IPO.h>
25 #include <llvm-c/Transforms/PassManagerBuilder.h>
26 #include <llvm-c/Transforms/Scalar.h>
27 #if LLVM_VERSION_MAJOR > 6
28 #include <llvm-c/Transforms/Utils.h>
29 #endif
30 
31 #include "jit/llvmjit.h"
32 #include "jit/llvmjit_emit.h"
33 #include "miscadmin.h"
34 #include "portability/instr_time.h"
35 #include "storage/ipc.h"
36 #include "utils/memutils.h"
37 #include "utils/resowner_private.h"
38 
39 /* Handle of a module emitted via ORC JIT */
40 typedef struct LLVMJitHandle
41 {
42  LLVMOrcJITStackRef stack;
43  LLVMOrcModuleHandle orc_handle;
45 
46 
47 /* types & functions commonly needed for JITing */
48 LLVMTypeRef TypeSizeT;
49 LLVMTypeRef TypeParamBool;
50 LLVMTypeRef TypeStorageBool;
51 LLVMTypeRef TypePGFunction;
52 LLVMTypeRef StructNullableDatum;
57 LLVMTypeRef StructHeapTupleData;
60 LLVMTypeRef StructBlockId;
62 LLVMTypeRef StructTupleConstr;
63 LLVMTypeRef StructTupleDescData;
68 LLVMTypeRef StructPGFinfoRecord;
69 LLVMTypeRef StructFmgrInfo;
71 LLVMTypeRef StructExprContext;
72 LLVMTypeRef StructExprEvalStep;
73 LLVMTypeRef StructExprState;
74 LLVMTypeRef StructAggState;
77 
78 LLVMValueRef AttributeTemplate;
79 
80 LLVMModuleRef llvm_types_module = NULL;
81 
82 static bool llvm_session_initialized = false;
83 static size_t llvm_generation = 0;
84 static const char *llvm_triple = NULL;
85 static const char *llvm_layout = NULL;
86 
87 
88 static LLVMTargetMachineRef llvm_opt0_targetmachine;
89 static LLVMTargetMachineRef llvm_opt3_targetmachine;
90 
91 static LLVMTargetRef llvm_targetref;
92 static LLVMOrcJITStackRef llvm_opt0_orc;
93 static LLVMOrcJITStackRef llvm_opt3_orc;
94 
95 
96 static void llvm_release_context(JitContext *context);
97 static void llvm_session_initialize(void);
98 static void llvm_shutdown(int code, Datum arg);
99 static void llvm_compile_module(LLVMJitContext *context);
100 static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
101 
102 static void llvm_create_types(void);
103 static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
104 
105 
107 
108 
109 /*
110  * Initialize LLVM JIT provider.
111  */
112 void
114 {
118 }
119 
120 /*
121  * Create a context for JITing work.
122  *
123  * The context, including subsidiary resources, will be cleaned up either when
124  * the context is explicitly released, or when the lifetime of
125  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
126  */
127 LLVMJitContext *
128 llvm_create_context(int jitFlags)
129 {
130  LLVMJitContext *context;
131 
133 
135 
137 
139  sizeof(LLVMJitContext));
140  context->base.flags = jitFlags;
141 
142  /* ensure cleanup */
143  context->base.resowner = CurrentResourceOwner;
145 
146  return context;
147 }
148 
149 /*
150  * Release resources required by one llvm context.
151  */
152 static void
154 {
155  LLVMJitContext *llvm_context = (LLVMJitContext *) context;
156 
158 
159  /*
160  * When this backend is exiting, don't clean up LLVM. As an error might
161  * have occurred from within LLVM, we do not want to risk reentering. All
162  * resource cleanup is going to happen through process exit.
163  */
165  {
166  if (llvm_context->module)
167  {
168  LLVMDisposeModule(llvm_context->module);
169  llvm_context->module = NULL;
170  }
171 
172  while (llvm_context->handles != NIL)
173  {
174  LLVMJitHandle *jit_handle;
175 
176  jit_handle = (LLVMJitHandle *) linitial(llvm_context->handles);
177  llvm_context->handles = list_delete_first(llvm_context->handles);
178 
179  LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
180  pfree(jit_handle);
181  }
182  }
183 }
184 
185 /*
186  * Return module which may be modified, e.g. by creating new functions.
187  */
188 LLVMModuleRef
189 llvm_mutable_module(LLVMJitContext *context)
190 {
192 
193  /*
194  * If there's no in-progress module, create a new one.
195  */
196  if (!context->module)
197  {
198  context->compiled = false;
199  context->module_generation = llvm_generation++;
200  context->module = LLVMModuleCreateWithName("pg");
201  LLVMSetTarget(context->module, llvm_triple);
202  LLVMSetDataLayout(context->module, llvm_layout);
203  }
204 
205  return context->module;
206 }
207 
208 /*
209  * Expand function name to be non-conflicting. This should be used by code
210  * generating code, when adding new externally visible function definitions to
211  * a Module.
212  */
213 char *
214 llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
215 {
216  Assert(context->module != NULL);
217 
218  context->base.instr.created_functions++;
219 
220  /*
221  * Previously we used dots to separate, but turns out some tools, e.g.
222  * GDB, don't like that and truncate name.
223  */
224  return psprintf("%s_%zu_%d",
225  basename,
226  context->module_generation,
227  context->counter++);
228 }
229 
230 /*
231  * Return pointer to function funcname, which has to exist. If there's pending
232  * code to be optimized and emitted, do so first.
233  */
234 void *
235 llvm_get_function(LLVMJitContext *context, const char *funcname)
236 {
237  LLVMOrcTargetAddress addr = 0;
238 #if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
239  ListCell *lc;
240 #endif
241 
243 
244  /*
245  * If there is a pending / not emitted module, compile and emit now.
246  * Otherwise we might not find the [correct] function.
247  */
248  if (!context->compiled)
249  {
250  llvm_compile_module(context);
251  }
252 
253  /*
254  * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
255  * to mangle here.
256  */
257 
258 #if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
259  foreach(lc, context->handles)
260  {
261  LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
262 
263  addr = 0;
264  if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
265  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
266  if (addr)
267  return (void *) (uintptr_t) addr;
268  }
269 
270 #else
271 
272 #if LLVM_VERSION_MAJOR < 5
273  if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
274  return (void *) (uintptr_t) addr;
275  if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
276  return (void *) (uintptr_t) addr;
277 #else
278  if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
279  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
280  if (addr)
281  return (void *) (uintptr_t) addr;
282  if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
283  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
284  if (addr)
285  return (void *) (uintptr_t) addr;
286 #endif /* LLVM_VERSION_MAJOR */
287 
288 #endif /* HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN */
289 
290  elog(ERROR, "failed to JIT: %s", funcname);
291 
292  return NULL;
293 }
294 
295 /*
296  * Return declaration for a function referenced in llvmjit_types.c, adding it
297  * to the module if necessary.
298  *
299  * This is used to make functions discovered via llvm_create_types() known to
300  * the module that's currently being worked on.
301  */
302 LLVMValueRef
303 llvm_pg_func(LLVMModuleRef mod, const char *funcname)
304 {
305  LLVMValueRef v_srcfn;
306  LLVMValueRef v_fn;
307 
308  /* don't repeatedly add function */
309  v_fn = LLVMGetNamedFunction(mod, funcname);
310  if (v_fn)
311  return v_fn;
312 
313  v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
314 
315  if (!v_srcfn)
316  elog(ERROR, "function %s not in llvmjit_types.c", funcname);
317 
318  v_fn = LLVMAddFunction(mod,
319  funcname,
320  LLVMGetElementType(LLVMTypeOf(v_srcfn)));
321  llvm_copy_attributes(v_srcfn, v_fn);
322 
323  return v_fn;
324 }
325 
326 /*
327  * Copy attributes from one function to another.
328  */
329 void
330 llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
331 {
332  int num_attributes;
333  int attno;
334  LLVMAttributeRef *attrs;
335 
336  num_attributes =
337  LLVMGetAttributeCountAtIndex(v_from, LLVMAttributeFunctionIndex);
338 
339  attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
340  LLVMGetAttributesAtIndex(v_from, LLVMAttributeFunctionIndex, attrs);
341 
342  for (attno = 0; attno < num_attributes; attno++)
343  {
344  LLVMAddAttributeAtIndex(v_to, LLVMAttributeFunctionIndex,
345  attrs[attno]);
346  }
347 }
348 
349 /*
350  * Return a callable LLVMValueRef for fcinfo.
351  */
352 LLVMValueRef
353 llvm_function_reference(LLVMJitContext *context,
354  LLVMBuilderRef builder,
355  LLVMModuleRef mod,
356  FunctionCallInfo fcinfo)
357 {
358  char *modname;
359  char *basename;
360  char *funcname;
361 
362  LLVMValueRef v_fn;
363 
364  fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
365 
366  if (modname != NULL && basename != NULL)
367  {
368  /* external function in loadable library */
369  funcname = psprintf("pgextern.%s.%s", modname, basename);
370  }
371  else if (basename != NULL)
372  {
373  /* internal function */
374  funcname = psprintf("%s", basename);
375  }
376  else
377  {
378  /*
379  * Function we don't know to handle, return pointer. We do so by
380  * creating a global constant containing a pointer to the function.
381  * Makes IR more readable.
382  */
383  LLVMValueRef v_fn_addr;
384 
385  funcname = psprintf("pgoidextern.%u",
386  fcinfo->flinfo->fn_oid);
387  v_fn = LLVMGetNamedGlobal(mod, funcname);
388  if (v_fn != 0)
389  return LLVMBuildLoad(builder, v_fn, "");
390 
391  v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
392 
393  v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
394  LLVMSetInitializer(v_fn, v_fn_addr);
395  LLVMSetGlobalConstant(v_fn, true);
396 
397  return LLVMBuildLoad(builder, v_fn, "");
398  }
399 
400  /* check if function already has been added */
401  v_fn = LLVMGetNamedFunction(mod, funcname);
402  if (v_fn != 0)
403  return v_fn;
404 
405  v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction));
406 
407  return v_fn;
408 }
409 
410 /*
411  * Optimize code in module using the flags set in context.
412  */
413 static void
414 llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
415 {
416  LLVMPassManagerBuilderRef llvm_pmb;
417  LLVMPassManagerRef llvm_mpm;
418  LLVMPassManagerRef llvm_fpm;
419  LLVMValueRef func;
420  int compile_optlevel;
421 
422  if (context->base.flags & PGJIT_OPT3)
423  compile_optlevel = 3;
424  else
425  compile_optlevel = 0;
426 
427  /*
428  * Have to create a new pass manager builder every pass through, as the
429  * inliner has some per-builder state. Otherwise one ends up only inlining
430  * a function the first time though.
431  */
432  llvm_pmb = LLVMPassManagerBuilderCreate();
433  LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
434  llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
435 
436  if (context->base.flags & PGJIT_OPT3)
437  {
438  /* TODO: Unscientifically determined threshold */
439  LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
440  }
441  else
442  {
443  /* we rely on mem2reg heavily, so emit even in the O0 case */
444  LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
445  }
446 
447  LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
448 
449  /*
450  * Do function level optimization. This could be moved to the point where
451  * functions are emitted, to reduce memory usage a bit.
452  */
453  LLVMInitializeFunctionPassManager(llvm_fpm);
454  for (func = LLVMGetFirstFunction(context->module);
455  func != NULL;
456  func = LLVMGetNextFunction(func))
457  LLVMRunFunctionPassManager(llvm_fpm, func);
458  LLVMFinalizeFunctionPassManager(llvm_fpm);
459  LLVMDisposePassManager(llvm_fpm);
460 
461  /*
462  * Perform module level optimization. We do so even in the non-optimized
463  * case, so always-inline functions etc get inlined. It's cheap enough.
464  */
465  llvm_mpm = LLVMCreatePassManager();
466  LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
467  llvm_mpm);
468  /* always use always-inliner pass */
469  if (!(context->base.flags & PGJIT_OPT3))
470  LLVMAddAlwaysInlinerPass(llvm_mpm);
471  /* if doing inlining, but no expensive optimization, add inlining pass */
472  if (context->base.flags & PGJIT_INLINE
473  && !(context->base.flags & PGJIT_OPT3))
474  LLVMAddFunctionInliningPass(llvm_mpm);
475  LLVMRunPassManager(llvm_mpm, context->module);
476  LLVMDisposePassManager(llvm_mpm);
477 
478  LLVMPassManagerBuilderDispose(llvm_pmb);
479 }
480 
481 /*
482  * Emit code for the currently pending module.
483  */
484 static void
485 llvm_compile_module(LLVMJitContext *context)
486 {
487  LLVMOrcModuleHandle orc_handle;
488  MemoryContext oldcontext;
489  static LLVMOrcJITStackRef compile_orc;
490  instr_time starttime;
491  instr_time endtime;
492 
493  if (context->base.flags & PGJIT_OPT3)
494  compile_orc = llvm_opt3_orc;
495  else
496  compile_orc = llvm_opt0_orc;
497 
498  /* perform inlining */
499  if (context->base.flags & PGJIT_INLINE)
500  {
501  INSTR_TIME_SET_CURRENT(starttime);
502  llvm_inline(context->module);
503  INSTR_TIME_SET_CURRENT(endtime);
504  INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
505  endtime, starttime);
506  }
507 
508  if (jit_dump_bitcode)
509  {
510  char *filename;
511 
512  filename = psprintf("%u.%zu.bc",
513  MyProcPid,
514  context->module_generation);
515  LLVMWriteBitcodeToFile(context->module, filename);
516  pfree(filename);
517  }
518 
519 
520  /* optimize according to the chosen optimization settings */
521  INSTR_TIME_SET_CURRENT(starttime);
522  llvm_optimize_module(context, context->module);
523  INSTR_TIME_SET_CURRENT(endtime);
524  INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
525  endtime, starttime);
526 
527  if (jit_dump_bitcode)
528  {
529  char *filename;
530 
531  filename = psprintf("%u.%zu.optimized.bc",
532  MyProcPid,
533  context->module_generation);
534  LLVMWriteBitcodeToFile(context->module, filename);
535  pfree(filename);
536  }
537 
538  /*
539  * Emit the code. Note that this can, depending on the optimization
540  * settings, take noticeable resources as code emission executes low-level
541  * instruction combining/selection passes etc. Without optimization a
542  * faster instruction selection mechanism is used.
543  */
544  INSTR_TIME_SET_CURRENT(starttime);
545 #if LLVM_VERSION_MAJOR > 6
546  {
547  if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, context->module,
548  llvm_resolve_symbol, NULL))
549  {
550  elog(ERROR, "failed to JIT module");
551  }
552 
553  /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
554  }
555 #elif LLVM_VERSION_MAJOR > 4
556  {
557  LLVMSharedModuleRef smod;
558 
559  smod = LLVMOrcMakeSharedModule(context->module);
560  if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, smod,
561  llvm_resolve_symbol, NULL))
562  {
563  elog(ERROR, "failed to JIT module");
564  }
565  LLVMOrcDisposeSharedModuleRef(smod);
566  }
567 #else /* LLVM 4.0 and 3.9 */
568  {
569  orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
570  llvm_resolve_symbol, NULL);
571  LLVMDisposeModule(context->module);
572  }
573 #endif
574  INSTR_TIME_SET_CURRENT(endtime);
575  INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
576  endtime, starttime);
577 
578  context->module = NULL;
579  context->compiled = true;
580 
581  /* remember emitted code for cleanup and lookups */
583  {
584  LLVMJitHandle *handle;
585 
586  handle = (LLVMJitHandle *) palloc(sizeof(LLVMJitHandle));
587  handle->stack = compile_orc;
588  handle->orc_handle = orc_handle;
589 
590  context->handles = lappend(context->handles, handle);
591  }
592  MemoryContextSwitchTo(oldcontext);
593 
594  ereport(DEBUG1,
595  (errmsg("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
596  INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
597  INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
598  INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
599  errhidestmt(true),
600  errhidecontext(true)));
601 }
602 
603 /*
604  * Per session initialization.
605  */
606 static void
608 {
609  MemoryContext oldcontext;
610  char *error = NULL;
611  char *cpu = NULL;
612  char *features = NULL;
613 
615  return;
616 
618 
619  LLVMInitializeNativeTarget();
620  LLVMInitializeNativeAsmPrinter();
621  LLVMInitializeNativeAsmParser();
622 
623  /*
624  * Synchronize types early, as that also includes inferring the target
625  * triple.
626  */
628 
629  if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
630  {
631  elog(FATAL, "failed to query triple %s\n", error);
632  }
633 
634  /*
635  * We want the generated code to use all available features. Therefore
636  * grab the host CPU string and detect features of the current CPU. The
637  * latter is needed because some CPU architectures default to enabling
638  * features not all CPUs have (weird, huh).
639  */
640  cpu = LLVMGetHostCPUName();
641  features = LLVMGetHostCPUFeatures();
642  elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
643  cpu, features);
644 
646  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
647  LLVMCodeGenLevelNone,
648  LLVMRelocDefault,
649  LLVMCodeModelJITDefault);
651  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
652  LLVMCodeGenLevelAggressive,
653  LLVMRelocDefault,
654  LLVMCodeModelJITDefault);
655 
656  LLVMDisposeMessage(cpu);
657  cpu = NULL;
658  LLVMDisposeMessage(features);
659  features = NULL;
660 
661  /* force symbols in main binary to be loaded */
662  LLVMLoadLibraryPermanently(NULL);
663 
664  llvm_opt0_orc = LLVMOrcCreateInstance(llvm_opt0_targetmachine);
665  llvm_opt3_orc = LLVMOrcCreateInstance(llvm_opt3_targetmachine);
666 
667 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
669  {
670  LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
671 
672  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
673  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
674  }
675 #endif
676 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
678  {
679  LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
680 
681  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
682  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
683  }
684 #endif
685 
687 
689 
690  MemoryContextSwitchTo(oldcontext);
691 }
692 
693 static void
695 {
696  /* unregister profiling support, needs to be flushed to be useful */
697 
698  if (llvm_opt3_orc)
699  {
700 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
702  LLVMOrcUnregisterPerf(llvm_opt3_orc);
703 #endif
704  LLVMOrcDisposeInstance(llvm_opt3_orc);
705  llvm_opt3_orc = NULL;
706  }
707 
708  if (llvm_opt0_orc)
709  {
710 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
712  LLVMOrcUnregisterPerf(llvm_opt0_orc);
713 #endif
714  LLVMOrcDisposeInstance(llvm_opt0_orc);
715  llvm_opt0_orc = NULL;
716  }
717 }
718 
719 /* helper for llvm_create_types, returning a global var's type */
720 static LLVMTypeRef
721 load_type(LLVMModuleRef mod, const char *name)
722 {
723  LLVMValueRef value;
724  LLVMTypeRef typ;
725 
726  /* this'll return a *pointer* to the global */
727  value = LLVMGetNamedGlobal(mod, name);
728  if (!value)
729  elog(ERROR, "type %s is unknown", name);
730 
731  /* therefore look at the contained type and return that */
732  typ = LLVMTypeOf(value);
733  Assert(typ != NULL);
734  typ = LLVMGetElementType(typ);
735  Assert(typ != NULL);
736  return typ;
737 }
738 
739 /* helper for llvm_create_types, returning a function's return type */
740 static LLVMTypeRef
741 load_return_type(LLVMModuleRef mod, const char *name)
742 {
743  LLVMValueRef value;
744  LLVMTypeRef typ;
745 
746  /* this'll return a *pointer* to the function */
747  value = LLVMGetNamedFunction(mod, name);
748  if (!value)
749  elog(ERROR, "function %s is unknown", name);
750 
751  /* get type of function pointer */
752  typ = LLVMTypeOf(value);
753  Assert(typ != NULL);
754  /* dereference pointer */
755  typ = LLVMGetElementType(typ);
756  Assert(typ != NULL);
757  /* and look at return type */
758  typ = LLVMGetReturnType(typ);
759  Assert(typ != NULL);
760 
761  return typ;
762 }
763 
764 /*
765  * Load required information, types, function signatures from llvmjit_types.c
766  * and make them available in global variables.
767  *
768  * Those global variables are then used while emitting code.
769  */
770 static void
772 {
773  char path[MAXPGPATH];
774  LLVMMemoryBufferRef buf;
775  char *msg;
776 
777  snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
778 
779  /* open file */
780  if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
781  {
782  elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
783  path, msg);
784  }
785 
786  /* eagerly load contents, going to need it all */
787  if (LLVMParseBitcode2(buf, &llvm_types_module))
788  {
789  elog(ERROR, "LLVMParseBitcode2 of %s failed", path);
790  }
791  LLVMDisposeMemoryBuffer(buf);
792 
793  /*
794  * Load triple & layout from clang emitted file so we're guaranteed to be
795  * compatible.
796  */
797  llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
798  llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
799 
800  TypeSizeT = load_type(llvm_types_module, "TypeSizeT");
801  TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
802  TypeStorageBool = load_type(llvm_types_module, "TypeStorageBool");
803  TypePGFunction = load_type(llvm_types_module, "TypePGFunction");
804  StructNullableDatum = load_type(llvm_types_module, "StructNullableDatum");
805  StructExprContext = load_type(llvm_types_module, "StructExprContext");
806  StructExprEvalStep = load_type(llvm_types_module, "StructExprEvalStep");
807  StructExprState = load_type(llvm_types_module, "StructExprState");
808  StructFunctionCallInfoData = load_type(llvm_types_module, "StructFunctionCallInfoData");
809  StructMemoryContextData = load_type(llvm_types_module, "StructMemoryContextData");
810  StructTupleTableSlot = load_type(llvm_types_module, "StructTupleTableSlot");
811  StructHeapTupleTableSlot = load_type(llvm_types_module, "StructHeapTupleTableSlot");
812  StructMinimalTupleTableSlot = load_type(llvm_types_module, "StructMinimalTupleTableSlot");
813  StructHeapTupleData = load_type(llvm_types_module, "StructHeapTupleData");
814  StructTupleDescData = load_type(llvm_types_module, "StructTupleDescData");
815  StructAggState = load_type(llvm_types_module, "StructAggState");
816  StructAggStatePerGroupData = load_type(llvm_types_module, "StructAggStatePerGroupData");
817  StructAggStatePerTransData = load_type(llvm_types_module, "StructAggStatePerTransData");
818 
819  AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
820 }
821 
822 /*
823  * Split a symbol into module / function parts. If the function is in the
824  * main binary (or an external library) *modname will be NULL.
825  */
826 void
827 llvm_split_symbol_name(const char *name, char **modname, char **funcname)
828 {
829  *modname = NULL;
830  *funcname = NULL;
831 
832  /*
833  * Module function names are pgextern.$module.$funcname
834  */
835  if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
836  {
837  /*
838  * Symbol names cannot contain a ., therefore we can split based on
839  * first and last occurrence of one.
840  */
841  *funcname = rindex(name, '.');
842  (*funcname)++; /* jump over . */
843 
844  *modname = pnstrdup(name + strlen("pgextern."),
845  *funcname - name - strlen("pgextern.") - 1);
846  Assert(funcname);
847 
848  *funcname = pstrdup(*funcname);
849  }
850  else
851  {
852  *modname = NULL;
853  *funcname = pstrdup(name);
854  }
855 }
856 
857 /*
858  * Attempt to resolve symbol, so LLVM can emit a reference to it.
859  */
860 static uint64_t
861 llvm_resolve_symbol(const char *symname, void *ctx)
862 {
863  uintptr_t addr;
864  char *funcname;
865  char *modname;
866 
867  /*
868  * macOS prefixes all object level symbols with an underscore. But neither
869  * dlsym() nor PG's inliner expect that. So undo.
870  */
871 #if defined(__darwin__)
872  if (symname[0] != '_')
873  elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
874  symname++;
875 #endif
876 
877  llvm_split_symbol_name(symname, &modname, &funcname);
878 
879  /* functions that aren't resolved to names shouldn't ever get here */
880  Assert(funcname);
881 
882  if (modname)
883  addr = (uintptr_t) load_external_function(modname, funcname,
884  true, NULL);
885  else
886  addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
887 
888  pfree(funcname);
889  if (modname)
890  pfree(modname);
891 
892  /* let LLVM will error out - should never happen */
893  if (!addr)
894  elog(WARNING, "failed to resolve name %s", symname);
895 
896  return (uint64_t) addr;
897 }
LLVMTypeRef StructPGFinfoRecord
Definition: llvmjit.c:68
#define NIL
Definition: pg_list.h:65
JitProviderResetAfterErrorCB reset_after_error
Definition: jit.h:75
LLVMTypeRef StructTupleConstr
Definition: llvmjit.c:62
#define DEBUG1
Definition: elog.h:25
LLVMTypeRef StructFunctionCallInfoData
Definition: llvmjit.c:70
int MyProcPid
Definition: globals.c:40
static void llvm_compile_module(LLVMJitContext *context)
Definition: llvmjit.c:485
char * pnstrdup(const char *in, Size len)
Definition: mcxt.c:1197
PG_MODULE_MAGIC
Definition: llvmjit.c:106
static LLVMTypeRef load_type(LLVMModuleRef mod, const char *name)
Definition: llvmjit.c:721
static void error(void)
Definition: sql-dyntest.c:147
LLVMTypeRef StructHeapTupleFieldsField3
Definition: llvmjit.c:53
LLVMTypeRef StructMinimalTupleData
Definition: llvmjit.c:58
LLVMOrcJITStackRef stack
Definition: llvmjit.c:42
PGFunction fn_addr
Definition: fmgr.h:58
void on_proc_exit(pg_on_exit_callback function, Datum arg)
Definition: ipc.c:305
LLVMTypeRef StructExprEvalStep
Definition: llvmjit.c:72
#define PointerGetDatum(X)
Definition: postgres.h:556
LLVMTypeRef TypeSizeT
Definition: llvmjit.c:48
LLVMTypeRef StructBlockId
Definition: llvmjit.c:60
LLVMTypeRef StructExprState
Definition: llvmjit.c:73
LLVMTypeRef StructHeapTupleFields
Definition: llvmjit.c:54
ResourceOwner CurrentResourceOwner
Definition: resowner.c:142
static LLVMTargetMachineRef llvm_opt3_targetmachine
Definition: llvmjit.c:89
char * pstrdup(const char *in)
Definition: mcxt.c:1186
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
bool jit_debugging_support
Definition: jit.c:34
struct timeval instr_time
Definition: instr_time.h:150
static void llvm_shutdown(int code, Datum arg)
Definition: llvmjit.c:694
LLVMTypeRef StructNullableDatum
Definition: llvmjit.c:52
LLVMTypeRef StructItemPointerData
Definition: llvmjit.c:59
Definition: jit.h:54
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
void _PG_jit_provider_init(JitProviderCallbacks *cb)
Definition: llvmjit.c:113
LLVMValueRef llvm_function_reference(LLVMJitContext *context, LLVMBuilderRef builder, LLVMModuleRef mod, FunctionCallInfo fcinfo)
Definition: llvmjit.c:353
int errhidestmt(bool hide_stmt)
Definition: elog.c:1142
#define INSTR_TIME_ACCUM_DIFF(x, y, z)
Definition: instr_time.h:182
static const char * llvm_layout
Definition: llvmjit.c:85
LLVMJitContext * llvm_create_context(int jitFlags)
Definition: llvmjit.c:128
LLVMTypeRef StructFmgrInfo
Definition: llvmjit.c:69
#define INSTR_TIME_GET_DOUBLE(t)
Definition: instr_time.h:199
LLVMTypeRef StructHeapTupleTableSlot
Definition: llvmjit.c:65
LLVMTypeRef StructAggState
Definition: llvmjit.c:74
static LLVMTypeRef load_return_type(LLVMModuleRef mod, const char *name)
Definition: llvmjit.c:741
void fmgr_symbol(Oid functionId, char **mod, char **fn)
Definition: fmgr.c:283
static const char * llvm_triple
Definition: llvmjit.c:84
LLVMTypeRef StructHeapTupleData
Definition: llvmjit.c:57
LLVMTypeRef StructHeapTupleDataChoice
Definition: llvmjit.c:56
#define PGJIT_OPT3
Definition: jit.h:21
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
Definition: llvmjit.c:827
void pfree(void *pointer)
Definition: mcxt.c:1056
#define linitial(l)
Definition: pg_list.h:195
#define ERROR
Definition: elog.h:43
struct LLVMJitHandle LLVMJitHandle
#define FATAL
Definition: elog.h:52
static uint64_t llvm_resolve_symbol(const char *name, void *ctx)
Definition: llvmjit.c:861
#define MAXPGPATH
LLVMOrcModuleHandle orc_handle
Definition: llvmjit.c:43
char * llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
Definition: llvmjit.c:214
#define DEBUG2
Definition: elog.h:24
void llvm_assert_in_fatal_section(void)
static char * buf
Definition: pg_test_fsync.c:67
LLVMTypeRef TypeStorageBool
Definition: llvmjit.c:50
LLVMTypeRef StructMemoryContextData
Definition: llvmjit.c:67
LLVMTypeRef StructExprContext
Definition: llvmjit.c:71
MemoryContext TopMemoryContext
Definition: mcxt.c:44
static LLVMOrcJITStackRef llvm_opt3_orc
Definition: llvmjit.c:93
void * load_external_function(const char *filename, const char *funcname, bool signalNotFound, void **filehandle)
Definition: dfmgr.c:107
List * lappend(List *list, void *datum)
Definition: list.c:321
LLVMTypeRef TypePGFunction
Definition: llvmjit.c:51
#define WARNING
Definition: elog.h:40
LLVMTypeRef StructAggStatePerTransData
Definition: llvmjit.c:76
JitProviderCompileExprCB compile_expr
Definition: jit.h:77
void ResourceOwnerRememberJIT(ResourceOwner owner, Datum handle)
Definition: resowner.c:1358
uintptr_t Datum
Definition: postgres.h:367
FmgrInfo * flinfo
Definition: fmgr.h:87
#define PGJIT_INLINE
Definition: jit.h:22
static bool llvm_session_initialized
Definition: llvmjit.c:82
void llvm_reset_after_error(void)
LLVMTypeRef StructMinimalTupleTableSlot
Definition: llvmjit.c:66
static size_t llvm_generation
Definition: llvmjit.c:83
void * MemoryContextAllocZero(MemoryContext context, Size size)
Definition: mcxt.c:839
LLVMTypeRef StructTupleTableSlot
Definition: llvmjit.c:64
static struct @143 value
Oid fn_oid
Definition: fmgr.h:59
#define ereport(elevel,...)
Definition: elog.h:144
LLVMModuleRef llvm_mutable_module(LLVMJitContext *context)
Definition: llvmjit.c:189
static LLVMTargetRef llvm_targetref
Definition: llvmjit.c:91
static void llvm_session_initialize(void)
Definition: llvmjit.c:607
static void llvm_release_context(JitContext *context)
Definition: llvmjit.c:153
#define Assert(condition)
Definition: c.h:745
#define lfirst(lc)
Definition: pg_list.h:190
bool proc_exit_inprogress
Definition: ipc.c:40
bool llvm_compile_expr(ExprState *state)
Definition: llvmjit_expr.c:78
LLVMTypeRef StructTupleDescData
Definition: llvmjit.c:63
LLVMTypeRef TypeParamBool
Definition: llvmjit.c:49
bool jit_profiling_support
Definition: jit.c:37
void * llvm_get_function(LLVMJitContext *context, const char *funcname)
Definition: llvmjit.c:235
bool jit_dump_bitcode
Definition: jit.c:35
#define INSTR_TIME_SET_CURRENT(t)
Definition: instr_time.h:156
const char * name
Definition: encode.c:561
static char * filename
Definition: pg_dumpall.c:90
void llvm_inline(LLVMModuleRef M)
static void llvm_create_types(void)
Definition: llvmjit.c:771
void * palloc(Size size)
Definition: mcxt.c:949
int errmsg(const char *fmt,...)
Definition: elog.c:824
static LLVMTargetMachineRef llvm_opt0_targetmachine
Definition: llvmjit.c:88
LLVMModuleRef llvm_types_module
Definition: llvmjit.c:80
void llvm_enter_fatal_on_oom(void)
#define elog(elevel,...)
Definition: elog.h:214
LLVMValueRef AttributeTemplate
Definition: llvmjit.c:78
void * arg
LLVMTypeRef StructHeapTupleHeaderData
Definition: llvmjit.c:55
void ResourceOwnerEnlargeJIT(ResourceOwner owner)
Definition: resowner.c:1347
void llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
Definition: llvmjit.c:330
static LLVMOrcJITStackRef llvm_opt0_orc
Definition: llvmjit.c:92
JitProviderReleaseContextCB release_context
Definition: jit.h:76
#define snprintf
Definition: port.h:193
static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
Definition: llvmjit.c:414
char pkglib_path[MAXPGPATH]
Definition: globals.c:73
int errhidecontext(bool hide_ctx)
Definition: elog.c:1161
List * list_delete_first(List *list)
Definition: list.c:860
LLVMTypeRef StructFormPgAttribute
Definition: llvmjit.c:61
LLVMValueRef llvm_pg_func(LLVMModuleRef mod, const char *funcname)
Definition: llvmjit.c:303
LLVMTypeRef StructAggStatePerGroupData
Definition: llvmjit.c:75