PostgreSQL Source Code  git master
llvmjit.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * llvmjit.c
4  * Core part of the LLVM JIT provider.
5  *
6  * Copyright (c) 2016-2019, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/jit/llvm/llvmjit.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres.h"
15 
16 #include "jit/llvmjit.h"
17 #include "jit/llvmjit_emit.h"
18 
19 #include "miscadmin.h"
20 
21 #include "utils/memutils.h"
22 #include "utils/resowner_private.h"
23 #include "portability/instr_time.h"
24 #include "storage/ipc.h"
25 
26 
27 #include <llvm-c/Analysis.h>
28 #include <llvm-c/BitReader.h>
29 #include <llvm-c/BitWriter.h>
30 #include <llvm-c/Core.h>
31 #include <llvm-c/ExecutionEngine.h>
32 #include <llvm-c/OrcBindings.h>
33 #include <llvm-c/Support.h>
34 #include <llvm-c/Target.h>
35 #include <llvm-c/Transforms/IPO.h>
36 #include <llvm-c/Transforms/PassManagerBuilder.h>
37 #include <llvm-c/Transforms/Scalar.h>
38 #if LLVM_VERSION_MAJOR > 6
39 #include <llvm-c/Transforms/Utils.h>
40 #endif
41 
42 
43 /* Handle of a module emitted via ORC JIT */
44 typedef struct LLVMJitHandle
45 {
46  LLVMOrcJITStackRef stack;
47  LLVMOrcModuleHandle orc_handle;
49 
50 
51 /* types & functions commonly needed for JITing */
52 LLVMTypeRef TypeSizeT;
53 LLVMTypeRef TypeParamBool;
54 LLVMTypeRef TypeStorageBool;
55 LLVMTypeRef TypePGFunction;
56 LLVMTypeRef StructNullableDatum;
61 LLVMTypeRef StructHeapTupleData;
64 LLVMTypeRef StructBlockId;
66 LLVMTypeRef StructTupleConstr;
67 LLVMTypeRef StructTupleDescData;
72 LLVMTypeRef StructPGFinfoRecord;
73 LLVMTypeRef StructFmgrInfo;
75 LLVMTypeRef StructExprContext;
76 LLVMTypeRef StructExprEvalStep;
77 LLVMTypeRef StructExprState;
78 LLVMTypeRef StructAggState;
81 
82 LLVMValueRef AttributeTemplate;
83 LLVMValueRef FuncStrlen;
84 LLVMValueRef FuncVarsizeAny;
89 LLVMValueRef FuncExecEvalSysVar;
91 LLVMValueRef FuncExecAggInitGroup;
92 
93 
94 static bool llvm_session_initialized = false;
95 static size_t llvm_generation = 0;
96 static const char *llvm_triple = NULL;
97 static const char *llvm_layout = NULL;
98 
99 
100 static LLVMTargetMachineRef llvm_opt0_targetmachine;
101 static LLVMTargetMachineRef llvm_opt3_targetmachine;
102 
103 static LLVMTargetRef llvm_targetref;
104 static LLVMOrcJITStackRef llvm_opt0_orc;
105 static LLVMOrcJITStackRef llvm_opt3_orc;
106 
107 
108 static void llvm_release_context(JitContext *context);
109 static void llvm_session_initialize(void);
110 static void llvm_shutdown(int code, Datum arg);
111 static void llvm_compile_module(LLVMJitContext *context);
112 static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
113 
114 static void llvm_create_types(void);
115 static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
116 
117 
119 
120 
121 /*
122  * Initialize LLVM JIT provider.
123  */
124 void
126 {
130 }
131 
132 /*
133  * Create a context for JITing work.
134  *
135  * The context, including subsidiary resources, will be cleaned up either when
136  * the context is explicitly released, or when the lifetime of
137  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
138  */
139 LLVMJitContext *
140 llvm_create_context(int jitFlags)
141 {
142  LLVMJitContext *context;
143 
145 
147 
149 
151  sizeof(LLVMJitContext));
152  context->base.flags = jitFlags;
153 
154  /* ensure cleanup */
155  context->base.resowner = CurrentResourceOwner;
157 
158  return context;
159 }
160 
161 /*
162  * Release resources required by one llvm context.
163  */
164 static void
166 {
167  LLVMJitContext *llvm_context = (LLVMJitContext *) context;
168 
170 
171  /*
172  * When this backend is exiting, don't clean up LLVM. As an error might
173  * have occurred from within LLVM, we do not want to risk reentering. All
174  * resource cleanup is going to happen through process exit.
175  */
177  {
178  if (llvm_context->module)
179  {
180  LLVMDisposeModule(llvm_context->module);
181  llvm_context->module = NULL;
182  }
183 
184  while (llvm_context->handles != NIL)
185  {
186  LLVMJitHandle *jit_handle;
187 
188  jit_handle = (LLVMJitHandle *) linitial(llvm_context->handles);
189  llvm_context->handles = list_delete_first(llvm_context->handles);
190 
191  LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
192  pfree(jit_handle);
193  }
194  }
195 }
196 
197 /*
198  * Return module which may be modified, e.g. by creating new functions.
199  */
200 LLVMModuleRef
201 llvm_mutable_module(LLVMJitContext *context)
202 {
204 
205  /*
206  * If there's no in-progress module, create a new one.
207  */
208  if (!context->module)
209  {
210  context->compiled = false;
211  context->module_generation = llvm_generation++;
212  context->module = LLVMModuleCreateWithName("pg");
213  LLVMSetTarget(context->module, llvm_triple);
214  LLVMSetDataLayout(context->module, llvm_layout);
215  }
216 
217  return context->module;
218 }
219 
220 /*
221  * Expand function name to be non-conflicting. This should be used by code
222  * generating code, when adding new externally visible function definitions to
223  * a Module.
224  */
225 char *
226 llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
227 {
228  Assert(context->module != NULL);
229 
230  context->base.instr.created_functions++;
231 
232  /*
233  * Previously we used dots to separate, but turns out some tools, e.g.
234  * GDB, don't like that and truncate name.
235  */
236  return psprintf("%s_%zu_%d",
237  basename,
238  context->module_generation,
239  context->counter++);
240 }
241 
242 /*
243  * Return pointer to function funcname, which has to exist. If there's pending
244  * code to be optimized and emitted, do so first.
245  */
246 void *
247 llvm_get_function(LLVMJitContext *context, const char *funcname)
248 {
249  LLVMOrcTargetAddress addr = 0;
250 #if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
251  ListCell *lc;
252 #endif
253 
255 
256  /*
257  * If there is a pending / not emitted module, compile and emit now.
258  * Otherwise we might not find the [correct] function.
259  */
260  if (!context->compiled)
261  {
262  llvm_compile_module(context);
263  }
264 
265  /*
266  * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
267  * to mangle here.
268  */
269 
270 #if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
271  foreach(lc, context->handles)
272  {
273  LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
274 
275  addr = 0;
276  if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
277  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
278  if (addr)
279  return (void *) (uintptr_t) addr;
280  }
281 
282 #else
283 
284 #if LLVM_VERSION_MAJOR < 5
285  if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
286  return (void *) (uintptr_t) addr;
287  if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
288  return (void *) (uintptr_t) addr;
289 #else
290  if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
291  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
292  if (addr)
293  return (void *) (uintptr_t) addr;
294  if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
295  elog(ERROR, "failed to look up symbol \"%s\"", funcname);
296  if (addr)
297  return (void *) (uintptr_t) addr;
298 #endif /* LLVM_VERSION_MAJOR */
299 
300 #endif /* HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN */
301 
302  elog(ERROR, "failed to JIT: %s", funcname);
303 
304  return NULL;
305 }
306 
307 /*
308  * Return declaration for passed function, adding it to the module if
309  * necessary.
310  *
311  * This is used to make functions imported by llvm_create_types() known to the
312  * module that's currently being worked on.
313  */
314 LLVMValueRef
315 llvm_get_decl(LLVMModuleRef mod, LLVMValueRef v_src)
316 {
317  LLVMValueRef v_fn;
318 
319  /* don't repeatedly add function */
320  v_fn = LLVMGetNamedFunction(mod, LLVMGetValueName(v_src));
321  if (v_fn)
322  return v_fn;
323 
324  v_fn = LLVMAddFunction(mod,
325  LLVMGetValueName(v_src),
326  LLVMGetElementType(LLVMTypeOf(v_src)));
327  llvm_copy_attributes(v_src, v_fn);
328 
329  return v_fn;
330 }
331 
332 /*
333  * Copy attributes from one function to another.
334  */
335 void
336 llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
337 {
338  int num_attributes;
339  int attno;
340  LLVMAttributeRef *attrs;
341 
342  num_attributes =
343  LLVMGetAttributeCountAtIndex(v_from, LLVMAttributeFunctionIndex);
344 
345  attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
346  LLVMGetAttributesAtIndex(v_from, LLVMAttributeFunctionIndex, attrs);
347 
348  for (attno = 0; attno < num_attributes; attno++)
349  {
350  LLVMAddAttributeAtIndex(v_to, LLVMAttributeFunctionIndex,
351  attrs[attno]);
352  }
353 }
354 
355 /*
356  * Return a callable LLVMValueRef for fcinfo.
357  */
358 LLVMValueRef
359 llvm_function_reference(LLVMJitContext *context,
360  LLVMBuilderRef builder,
361  LLVMModuleRef mod,
362  FunctionCallInfo fcinfo)
363 {
364  char *modname;
365  char *basename;
366  char *funcname;
367 
368  LLVMValueRef v_fn;
369 
370  fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
371 
372  if (modname != NULL && basename != NULL)
373  {
374  /* external function in loadable library */
375  funcname = psprintf("pgextern.%s.%s", modname, basename);
376  }
377  else if (basename != NULL)
378  {
379  /* internal function */
380  funcname = psprintf("%s", basename);
381  }
382  else
383  {
384  /*
385  * Function we don't know to handle, return pointer. We do so by
386  * creating a global constant containing a pointer to the function.
387  * Makes IR more readable.
388  */
389  LLVMValueRef v_fn_addr;
390 
391  funcname = psprintf("pgoidextern.%u",
392  fcinfo->flinfo->fn_oid);
393  v_fn = LLVMGetNamedGlobal(mod, funcname);
394  if (v_fn != 0)
395  return LLVMBuildLoad(builder, v_fn, "");
396 
397  v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
398 
399  v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
400  LLVMSetInitializer(v_fn, v_fn_addr);
401  LLVMSetGlobalConstant(v_fn, true);
402 
403  return LLVMBuildLoad(builder, v_fn, "");
404  }
405 
406  /* check if function already has been added */
407  v_fn = LLVMGetNamedFunction(mod, funcname);
408  if (v_fn != 0)
409  return v_fn;
410 
411  v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction));
412 
413  return v_fn;
414 }
415 
416 /*
417  * Optimize code in module using the flags set in context.
418  */
419 static void
420 llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
421 {
422  LLVMPassManagerBuilderRef llvm_pmb;
423  LLVMPassManagerRef llvm_mpm;
424  LLVMPassManagerRef llvm_fpm;
425  LLVMValueRef func;
426  int compile_optlevel;
427 
428  if (context->base.flags & PGJIT_OPT3)
429  compile_optlevel = 3;
430  else
431  compile_optlevel = 0;
432 
433  /*
434  * Have to create a new pass manager builder every pass through, as the
435  * inliner has some per-builder state. Otherwise one ends up only inlining
436  * a function the first time though.
437  */
438  llvm_pmb = LLVMPassManagerBuilderCreate();
439  LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
440  llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
441 
442  if (context->base.flags & PGJIT_OPT3)
443  {
444  /* TODO: Unscientifically determined threshold */
445  LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
446  }
447  else
448  {
449  /* we rely on mem2reg heavily, so emit even in the O0 case */
450  LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
451  }
452 
453  LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
454 
455  /*
456  * Do function level optimization. This could be moved to the point where
457  * functions are emitted, to reduce memory usage a bit.
458  */
459  LLVMInitializeFunctionPassManager(llvm_fpm);
460  for (func = LLVMGetFirstFunction(context->module);
461  func != NULL;
462  func = LLVMGetNextFunction(func))
463  LLVMRunFunctionPassManager(llvm_fpm, func);
464  LLVMFinalizeFunctionPassManager(llvm_fpm);
465  LLVMDisposePassManager(llvm_fpm);
466 
467  /*
468  * Perform module level optimization. We do so even in the non-optimized
469  * case, so always-inline functions etc get inlined. It's cheap enough.
470  */
471  llvm_mpm = LLVMCreatePassManager();
472  LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
473  llvm_mpm);
474  /* always use always-inliner pass */
475  if (!(context->base.flags & PGJIT_OPT3))
476  LLVMAddAlwaysInlinerPass(llvm_mpm);
477  /* if doing inlining, but no expensive optimization, add inlining pass */
478  if (context->base.flags & PGJIT_INLINE
479  && !(context->base.flags & PGJIT_OPT3))
480  LLVMAddFunctionInliningPass(llvm_mpm);
481  LLVMRunPassManager(llvm_mpm, context->module);
482  LLVMDisposePassManager(llvm_mpm);
483 
484  LLVMPassManagerBuilderDispose(llvm_pmb);
485 }
486 
487 /*
488  * Emit code for the currently pending module.
489  */
490 static void
491 llvm_compile_module(LLVMJitContext *context)
492 {
493  LLVMOrcModuleHandle orc_handle;
494  MemoryContext oldcontext;
495  static LLVMOrcJITStackRef compile_orc;
496  instr_time starttime;
497  instr_time endtime;
498 
499  if (context->base.flags & PGJIT_OPT3)
500  compile_orc = llvm_opt3_orc;
501  else
502  compile_orc = llvm_opt0_orc;
503 
504  /* perform inlining */
505  if (context->base.flags & PGJIT_INLINE)
506  {
507  INSTR_TIME_SET_CURRENT(starttime);
508  llvm_inline(context->module);
509  INSTR_TIME_SET_CURRENT(endtime);
510  INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
511  endtime, starttime);
512  }
513 
514  if (jit_dump_bitcode)
515  {
516  char *filename;
517 
518  filename = psprintf("%u.%zu.bc",
519  MyProcPid,
520  context->module_generation);
521  LLVMWriteBitcodeToFile(context->module, filename);
522  pfree(filename);
523  }
524 
525 
526  /* optimize according to the chosen optimization settings */
527  INSTR_TIME_SET_CURRENT(starttime);
528  llvm_optimize_module(context, context->module);
529  INSTR_TIME_SET_CURRENT(endtime);
530  INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
531  endtime, starttime);
532 
533  if (jit_dump_bitcode)
534  {
535  char *filename;
536 
537  filename = psprintf("%u.%zu.optimized.bc",
538  MyProcPid,
539  context->module_generation);
540  LLVMWriteBitcodeToFile(context->module, filename);
541  pfree(filename);
542  }
543 
544  /*
545  * Emit the code. Note that this can, depending on the optimization
546  * settings, take noticeable resources as code emission executes low-level
547  * instruction combining/selection passes etc. Without optimization a
548  * faster instruction selection mechanism is used.
549  */
550  INSTR_TIME_SET_CURRENT(starttime);
551 #if LLVM_VERSION_MAJOR > 6
552  {
553  if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, context->module,
554  llvm_resolve_symbol, NULL))
555  {
556  elog(ERROR, "failed to JIT module");
557  }
558 
559  /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
560  }
561 #elif LLVM_VERSION_MAJOR > 4
562  {
563  LLVMSharedModuleRef smod;
564 
565  smod = LLVMOrcMakeSharedModule(context->module);
566  if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, smod,
567  llvm_resolve_symbol, NULL))
568  {
569  elog(ERROR, "failed to JIT module");
570  }
571  LLVMOrcDisposeSharedModuleRef(smod);
572  }
573 #else /* LLVM 4.0 and 3.9 */
574  {
575  orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
576  llvm_resolve_symbol, NULL);
577  LLVMDisposeModule(context->module);
578  }
579 #endif
580  INSTR_TIME_SET_CURRENT(endtime);
581  INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
582  endtime, starttime);
583 
584  context->module = NULL;
585  context->compiled = true;
586 
587  /* remember emitted code for cleanup and lookups */
589  {
590  LLVMJitHandle *handle;
591 
592  handle = (LLVMJitHandle *) palloc(sizeof(LLVMJitHandle));
593  handle->stack = compile_orc;
594  handle->orc_handle = orc_handle;
595 
596  context->handles = lappend(context->handles, handle);
597  }
598  MemoryContextSwitchTo(oldcontext);
599 
600  ereport(DEBUG1,
601  (errmsg("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
602  INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
603  INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
604  INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
605  errhidestmt(true),
606  errhidecontext(true)));
607 }
608 
609 /*
610  * Per session initialization.
611  */
612 static void
614 {
615  MemoryContext oldcontext;
616  char *error = NULL;
617  char *cpu = NULL;
618  char *features = NULL;
619 
621  return;
622 
624 
625  LLVMInitializeNativeTarget();
626  LLVMInitializeNativeAsmPrinter();
627  LLVMInitializeNativeAsmParser();
628 
629  /*
630  * Synchronize types early, as that also includes inferring the target
631  * triple.
632  */
634 
635  if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
636  {
637  elog(FATAL, "failed to query triple %s\n", error);
638  }
639 
640  /*
641  * We want the generated code to use all available features. Therefore
642  * grab the host CPU string and detect features of the current CPU. The
643  * latter is needed because some CPU architectures default to enabling
644  * features not all CPUs have (weird, huh).
645  */
646  cpu = LLVMGetHostCPUName();
647  features = LLVMGetHostCPUFeatures();
648  elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
649  cpu, features);
650 
652  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
653  LLVMCodeGenLevelNone,
654  LLVMRelocDefault,
655  LLVMCodeModelJITDefault);
657  LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
658  LLVMCodeGenLevelAggressive,
659  LLVMRelocDefault,
660  LLVMCodeModelJITDefault);
661 
662  LLVMDisposeMessage(cpu);
663  cpu = NULL;
664  LLVMDisposeMessage(features);
665  features = NULL;
666 
667  /* force symbols in main binary to be loaded */
668  LLVMLoadLibraryPermanently(NULL);
669 
670  llvm_opt0_orc = LLVMOrcCreateInstance(llvm_opt0_targetmachine);
671  llvm_opt3_orc = LLVMOrcCreateInstance(llvm_opt3_targetmachine);
672 
673 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
675  {
676  LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
677 
678  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
679  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
680  }
681 #endif
682 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
684  {
685  LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
686 
687  LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
688  LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
689  }
690 #endif
691 
693 
695 
696  MemoryContextSwitchTo(oldcontext);
697 }
698 
699 static void
701 {
702  /* unregister profiling support, needs to be flushed to be useful */
703 
704  if (llvm_opt3_orc)
705  {
706 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
708  LLVMOrcUnregisterPerf(llvm_opt3_orc);
709 #endif
710  LLVMOrcDisposeInstance(llvm_opt3_orc);
711  llvm_opt3_orc = NULL;
712  }
713 
714  if (llvm_opt0_orc)
715  {
716 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
718  LLVMOrcUnregisterPerf(llvm_opt0_orc);
719 #endif
720  LLVMOrcDisposeInstance(llvm_opt0_orc);
721  llvm_opt0_orc = NULL;
722  }
723 }
724 
725 /* helper for llvm_create_types, returning a global var's type */
726 static LLVMTypeRef
727 load_type(LLVMModuleRef mod, const char *name)
728 {
729  LLVMValueRef value;
730  LLVMTypeRef typ;
731 
732  /* this'll return a *pointer* to the global */
733  value = LLVMGetNamedGlobal(mod, name);
734  if (!value)
735  elog(ERROR, "type %s is unknown", name);
736 
737  /* therefore look at the contained type and return that */
738  typ = LLVMTypeOf(value);
739  Assert(typ != NULL);
740  typ = LLVMGetElementType(typ);
741  Assert(typ != NULL);
742  return typ;
743 }
744 
745 /* helper for llvm_create_types, returning a function's return type */
746 static LLVMTypeRef
747 load_return_type(LLVMModuleRef mod, const char *name)
748 {
749  LLVMValueRef value;
750  LLVMTypeRef typ;
751 
752  /* this'll return a *pointer* to the function */
753  value = LLVMGetNamedFunction(mod, name);
754  if (!value)
755  elog(ERROR, "function %s is unknown", name);
756 
757  /* get type of function pointer */
758  typ = LLVMTypeOf(value);
759  Assert(typ != NULL);
760  /* dereference pointer */
761  typ = LLVMGetElementType(typ);
762  Assert(typ != NULL);
763  /* and look at return type */
764  typ = LLVMGetReturnType(typ);
765  Assert(typ != NULL);
766 
767  return typ;
768 }
769 
770 /*
771  * Load required information, types, function signatures from llvmjit_types.c
772  * and make them available in global variables.
773  *
774  * Those global variables are then used while emitting code.
775  */
776 static void
778 {
779  char path[MAXPGPATH];
780  LLVMMemoryBufferRef buf;
781  char *msg;
782  LLVMModuleRef mod = NULL;
783 
784  snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
785 
786  /* open file */
787  if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
788  {
789  elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
790  path, msg);
791  }
792 
793  /* eagerly load contents, going to need it all */
794  if (LLVMParseBitcode2(buf, &mod))
795  {
796  elog(ERROR, "LLVMParseBitcode2 of %s failed", path);
797  }
798  LLVMDisposeMemoryBuffer(buf);
799 
800  /*
801  * Load triple & layout from clang emitted file so we're guaranteed to be
802  * compatible.
803  */
804  llvm_triple = pstrdup(LLVMGetTarget(mod));
805  llvm_layout = pstrdup(LLVMGetDataLayoutStr(mod));
806 
807  TypeSizeT = load_type(mod, "TypeSizeT");
808  TypeParamBool = load_return_type(mod, "FunctionReturningBool");
809  TypeStorageBool = load_type(mod, "TypeStorageBool");
810  TypePGFunction = load_type(mod, "TypePGFunction");
811  StructNullableDatum = load_type(mod, "StructNullableDatum");
812  StructExprContext = load_type(mod, "StructExprContext");
813  StructExprEvalStep = load_type(mod, "StructExprEvalStep");
814  StructExprState = load_type(mod, "StructExprState");
815  StructFunctionCallInfoData = load_type(mod, "StructFunctionCallInfoData");
816  StructMemoryContextData = load_type(mod, "StructMemoryContextData");
817  StructTupleTableSlot = load_type(mod, "StructTupleTableSlot");
818  StructHeapTupleTableSlot = load_type(mod, "StructHeapTupleTableSlot");
819  StructMinimalTupleTableSlot = load_type(mod, "StructMinimalTupleTableSlot");
820  StructHeapTupleData = load_type(mod, "StructHeapTupleData");
821  StructTupleDescData = load_type(mod, "StructTupleDescData");
822  StructAggState = load_type(mod, "StructAggState");
823  StructAggStatePerGroupData = load_type(mod, "StructAggStatePerGroupData");
824  StructAggStatePerTransData = load_type(mod, "StructAggStatePerTransData");
825 
826  AttributeTemplate = LLVMGetNamedFunction(mod, "AttributeTemplate");
827  FuncStrlen = LLVMGetNamedFunction(mod, "strlen");
828  FuncVarsizeAny = LLVMGetNamedFunction(mod, "varsize_any");
829  FuncSlotGetsomeattrsInt = LLVMGetNamedFunction(mod, "slot_getsomeattrs_int");
830  FuncSlotGetmissingattrs = LLVMGetNamedFunction(mod, "slot_getmissingattrs");
831  FuncMakeExpandedObjectReadOnlyInternal = LLVMGetNamedFunction(mod, "MakeExpandedObjectReadOnlyInternal");
832  FuncExecEvalSubscriptingRef = LLVMGetNamedFunction(mod, "ExecEvalSubscriptingRef");
833  FuncExecEvalSysVar = LLVMGetNamedFunction(mod, "ExecEvalSysVar");
834  FuncExecAggTransReparent = LLVMGetNamedFunction(mod, "ExecAggTransReparent");
835  FuncExecAggInitGroup = LLVMGetNamedFunction(mod, "ExecAggInitGroup");
836 
837  /*
838  * Leave the module alive, otherwise references to function would be
839  * dangling.
840  */
841 
842  return;
843 }
844 
845 /*
846  * Split a symbol into module / function parts. If the function is in the
847  * main binary (or an external library) *modname will be NULL.
848  */
849 void
850 llvm_split_symbol_name(const char *name, char **modname, char **funcname)
851 {
852  *modname = NULL;
853  *funcname = NULL;
854 
855  /*
856  * Module function names are pgextern.$module.$funcname
857  */
858  if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
859  {
860  /*
861  * Symbol names cannot contain a ., therefore we can split based on
862  * first and last occurrence of one.
863  */
864  *funcname = rindex(name, '.');
865  (*funcname)++; /* jump over . */
866 
867  *modname = pnstrdup(name + strlen("pgextern."),
868  *funcname - name - strlen("pgextern.") - 1);
869  Assert(funcname);
870 
871  *funcname = pstrdup(*funcname);
872  }
873  else
874  {
875  *modname = NULL;
876  *funcname = pstrdup(name);
877  }
878 }
879 
880 /*
881  * Attempt to resolve symbol, so LLVM can emit a reference to it.
882  */
883 static uint64_t
884 llvm_resolve_symbol(const char *symname, void *ctx)
885 {
886  uintptr_t addr;
887  char *funcname;
888  char *modname;
889 
890  /*
891  * macOS prefixes all object level symbols with an underscore. But neither
892  * dlsym() nor PG's inliner expect that. So undo.
893  */
894 #if defined(__darwin__)
895  if (symname[0] != '_')
896  elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
897  symname++;
898 #endif
899 
900  llvm_split_symbol_name(symname, &modname, &funcname);
901 
902  /* functions that aren't resolved to names shouldn't ever get here */
903  Assert(funcname);
904 
905  if (modname)
906  addr = (uintptr_t) load_external_function(modname, funcname,
907  true, NULL);
908  else
909  addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
910 
911  pfree(funcname);
912  if (modname)
913  pfree(modname);
914 
915  /* let LLVM will error out - should never happen */
916  if (!addr)
917  elog(WARNING, "failed to resolve name %s", symname);
918 
919  return (uint64_t) addr;
920 }
LLVMTypeRef StructPGFinfoRecord
Definition: llvmjit.c:72
#define NIL
Definition: pg_list.h:65
JitProviderResetAfterErrorCB reset_after_error
Definition: jit.h:75
LLVMTypeRef StructTupleConstr
Definition: llvmjit.c:66
#define DEBUG1
Definition: elog.h:25
LLVMTypeRef StructFunctionCallInfoData
Definition: llvmjit.c:74
int MyProcPid
Definition: globals.c:40
static void llvm_compile_module(LLVMJitContext *context)
Definition: llvmjit.c:491
char * pnstrdup(const char *in, Size len)
Definition: mcxt.c:1172
PG_MODULE_MAGIC
Definition: llvmjit.c:118
static LLVMTypeRef load_type(LLVMModuleRef mod, const char *name)
Definition: llvmjit.c:727
static void error(void)
Definition: sql-dyntest.c:147
LLVMTypeRef StructHeapTupleFieldsField3
Definition: llvmjit.c:57
LLVMTypeRef StructMinimalTupleData
Definition: llvmjit.c:62
LLVMOrcJITStackRef stack
Definition: llvmjit.c:46
PGFunction fn_addr
Definition: fmgr.h:58
LLVMTypeRef StructExprEvalStep
Definition: llvmjit.c:76
#define PointerGetDatum(X)
Definition: postgres.h:556
LLVMTypeRef TypeSizeT
Definition: llvmjit.c:52
LLVMTypeRef StructBlockId
Definition: llvmjit.c:64
LLVMTypeRef StructExprState
Definition: llvmjit.c:77
LLVMTypeRef StructHeapTupleFields
Definition: llvmjit.c:58
ResourceOwner CurrentResourceOwner
Definition: resowner.c:142
static LLVMTargetMachineRef llvm_opt3_targetmachine
Definition: llvmjit.c:101
char * pstrdup(const char *in)
Definition: mcxt.c:1161
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
bool jit_debugging_support
Definition: jit.c:37
LLVMValueRef FuncVarsizeAny
Definition: llvmjit.c:84
struct timeval instr_time
Definition: instr_time.h:150
static void llvm_shutdown(int code, Datum arg)
Definition: llvmjit.c:700
LLVMTypeRef StructNullableDatum
Definition: llvmjit.c:56
LLVMTypeRef StructItemPointerData
Definition: llvmjit.c:63
static struct @144 value
LLVMValueRef llvm_get_decl(LLVMModuleRef mod, LLVMValueRef v_src)
Definition: llvmjit.c:315
Definition: jit.h:54
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
void _PG_jit_provider_init(JitProviderCallbacks *cb)
Definition: llvmjit.c:125
LLVMValueRef llvm_function_reference(LLVMJitContext *context, LLVMBuilderRef builder, LLVMModuleRef mod, FunctionCallInfo fcinfo)
Definition: llvmjit.c:359
int errhidestmt(bool hide_stmt)
Definition: elog.c:1055
#define INSTR_TIME_ACCUM_DIFF(x, y, z)
Definition: instr_time.h:182
static const char * llvm_layout
Definition: llvmjit.c:97
LLVMJitContext * llvm_create_context(int jitFlags)
Definition: llvmjit.c:140
LLVMTypeRef StructFmgrInfo
Definition: llvmjit.c:73
#define INSTR_TIME_GET_DOUBLE(t)
Definition: instr_time.h:199
LLVMTypeRef StructHeapTupleTableSlot
Definition: llvmjit.c:69
LLVMTypeRef StructAggState
Definition: llvmjit.c:78
static LLVMTypeRef load_return_type(LLVMModuleRef mod, const char *name)
Definition: llvmjit.c:747
void fmgr_symbol(Oid functionId, char **mod, char **fn)
Definition: fmgr.c:281
static const char * llvm_triple
Definition: llvmjit.c:96
LLVMTypeRef StructHeapTupleData
Definition: llvmjit.c:61
LLVMTypeRef StructHeapTupleDataChoice
Definition: llvmjit.c:60
#define PGJIT_OPT3
Definition: jit.h:21
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
Definition: llvmjit.c:850
void pfree(void *pointer)
Definition: mcxt.c:1031
#define linitial(l)
Definition: pg_list.h:195
#define ERROR
Definition: elog.h:43
struct LLVMJitHandle LLVMJitHandle
#define FATAL
Definition: elog.h:52
static uint64_t llvm_resolve_symbol(const char *name, void *ctx)
Definition: llvmjit.c:884
#define MAXPGPATH
LLVMValueRef FuncExecAggInitGroup
Definition: llvmjit.c:91
LLVMOrcModuleHandle orc_handle
Definition: llvmjit.c:47
char * llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
Definition: llvmjit.c:226
#define DEBUG2
Definition: elog.h:24
void llvm_assert_in_fatal_section(void)
static char * buf
Definition: pg_test_fsync.c:68
LLVMTypeRef TypeStorageBool
Definition: llvmjit.c:54
LLVMValueRef FuncExecEvalSubscriptingRef
Definition: llvmjit.c:88
LLVMValueRef FuncMakeExpandedObjectReadOnlyInternal
Definition: llvmjit.c:87
void before_shmem_exit(pg_on_exit_callback function, Datum arg)
Definition: ipc.c:333
LLVMValueRef FuncSlotGetsomeattrsInt
Definition: llvmjit.c:85
LLVMTypeRef StructMemoryContextData
Definition: llvmjit.c:71
PGFunction load_external_function(const char *filename, const char *funcname, bool signalNotFound, void **filehandle)
Definition: dfmgr.c:107
LLVMTypeRef StructExprContext
Definition: llvmjit.c:75
#define ereport(elevel, rest)
Definition: elog.h:141
MemoryContext TopMemoryContext
Definition: mcxt.c:44
static LLVMOrcJITStackRef llvm_opt3_orc
Definition: llvmjit.c:105
List * lappend(List *list, void *datum)
Definition: list.c:321
LLVMTypeRef TypePGFunction
Definition: llvmjit.c:55
#define WARNING
Definition: elog.h:40
LLVMTypeRef StructAggStatePerTransData
Definition: llvmjit.c:80
JitProviderCompileExprCB compile_expr
Definition: jit.h:77
void ResourceOwnerRememberJIT(ResourceOwner owner, Datum handle)
Definition: resowner.c:1334
uintptr_t Datum
Definition: postgres.h:367
FmgrInfo * flinfo
Definition: fmgr.h:87
#define PGJIT_INLINE
Definition: jit.h:22
LLVMValueRef FuncSlotGetmissingattrs
Definition: llvmjit.c:86
static bool llvm_session_initialized
Definition: llvmjit.c:94
void llvm_reset_after_error(void)
LLVMValueRef FuncStrlen
Definition: llvmjit.c:83
LLVMTypeRef StructMinimalTupleTableSlot
Definition: llvmjit.c:70
static size_t llvm_generation
Definition: llvmjit.c:95
void * MemoryContextAllocZero(MemoryContext context, Size size)
Definition: mcxt.c:814
LLVMTypeRef StructTupleTableSlot
Definition: llvmjit.c:68
LLVMValueRef FuncExecEvalSysVar
Definition: llvmjit.c:89
Oid fn_oid
Definition: fmgr.h:59
LLVMModuleRef llvm_mutable_module(LLVMJitContext *context)
Definition: llvmjit.c:201
static LLVMTargetRef llvm_targetref
Definition: llvmjit.c:103
static void llvm_session_initialize(void)
Definition: llvmjit.c:613
static void llvm_release_context(JitContext *context)
Definition: llvmjit.c:165
#define Assert(condition)
Definition: c.h:732
#define lfirst(lc)
Definition: pg_list.h:190
bool proc_exit_inprogress
Definition: ipc.c:40
bool llvm_compile_expr(ExprState *state)
Definition: llvmjit_expr.c:73
LLVMTypeRef StructTupleDescData
Definition: llvmjit.c:67
LLVMTypeRef TypeParamBool
Definition: llvmjit.c:53
bool jit_profiling_support
Definition: jit.c:40
void * llvm_get_function(LLVMJitContext *context, const char *funcname)
Definition: llvmjit.c:247
bool jit_dump_bitcode
Definition: jit.c:38
#define INSTR_TIME_SET_CURRENT(t)
Definition: instr_time.h:156
const char * name
Definition: encode.c:521
static char * filename
Definition: pg_dumpall.c:91
void llvm_inline(LLVMModuleRef M)
static void llvm_create_types(void)
Definition: llvmjit.c:777
void * palloc(Size size)
Definition: mcxt.c:924
int errmsg(const char *fmt,...)
Definition: elog.c:784
static LLVMTargetMachineRef llvm_opt0_targetmachine
Definition: llvmjit.c:100
void llvm_enter_fatal_on_oom(void)
#define elog(elevel,...)
Definition: elog.h:226
LLVMValueRef AttributeTemplate
Definition: llvmjit.c:82
void * arg
LLVMTypeRef StructHeapTupleHeaderData
Definition: llvmjit.c:59
void ResourceOwnerEnlargeJIT(ResourceOwner owner)
Definition: resowner.c:1323
void llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
Definition: llvmjit.c:336
static LLVMOrcJITStackRef llvm_opt0_orc
Definition: llvmjit.c:104
JitProviderReleaseContextCB release_context
Definition: jit.h:76
#define snprintf
Definition: port.h:192
static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
Definition: llvmjit.c:420
char pkglib_path[MAXPGPATH]
Definition: globals.c:73
LLVMValueRef FuncExecAggTransReparent
Definition: llvmjit.c:90
int errhidecontext(bool hide_ctx)
Definition: elog.c:1074
List * list_delete_first(List *list)
Definition: list.c:857
LLVMTypeRef StructFormPgAttribute
Definition: llvmjit.c:65
LLVMTypeRef StructAggStatePerGroupData
Definition: llvmjit.c:79