16 #include <llvm-c/Analysis.h>
17 #include <llvm-c/BitReader.h>
18 #include <llvm-c/BitWriter.h>
19 #include <llvm-c/Core.h>
20 #include <llvm-c/ExecutionEngine.h>
21 #if LLVM_VERSION_MAJOR > 16
22 #include <llvm-c/Transforms/PassBuilder.h>
24 #if LLVM_VERSION_MAJOR > 11
25 #include <llvm-c/Orc.h>
26 #include <llvm-c/OrcEE.h>
27 #include <llvm-c/LLJIT.h>
29 #include <llvm-c/OrcBindings.h>
31 #include <llvm-c/Support.h>
32 #include <llvm-c/Target.h>
33 #if LLVM_VERSION_MAJOR < 17
34 #include <llvm-c/Transforms/IPO.h>
35 #include <llvm-c/Transforms/PassManagerBuilder.h>
36 #include <llvm-c/Transforms/Scalar.h>
37 #if LLVM_VERSION_MAJOR > 6
38 #include <llvm-c/Transforms/Utils.h>
50 #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
55 #if LLVM_VERSION_MAJOR > 11
56 LLVMOrcLLJITRef lljit;
57 LLVMOrcResourceTrackerRef resource_tracker;
108 #if LLVM_VERSION_MAJOR > 11
109 static LLVMOrcThreadSafeContextRef llvm_ts_context;
129 #if LLVM_VERSION_MAJOR > 11
130 static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef
tm);
131 static char *llvm_error_message(LLVMErrorRef
error);
139 .
name =
"LLVM JIT context",
190 elog(
ERROR,
"Trying to recreate a non-existing context");
239 LLVMJitContext *context;
250 sizeof(LLVMJitContext));
251 context->base.flags = jitFlags;
268 LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
287 if (llvm_jit_context->module)
289 LLVMDisposeModule(llvm_jit_context->module);
290 llvm_jit_context->module = NULL;
293 foreach(lc, llvm_jit_context->handles)
297 #if LLVM_VERSION_MAJOR > 11
299 LLVMOrcExecutionSessionRef ee;
300 LLVMOrcSymbolStringPoolRef sp;
302 LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
303 LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
311 ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
312 sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
313 LLVMOrcSymbolStringPoolClearDeadEntries(sp);
324 llvm_jit_context->handles =
NIL;
343 if (!context->module)
345 context->compiled =
false;
347 context->module = LLVMModuleCreateWithNameInContext(
"pg",
llvm_context);
352 return context->module;
363 Assert(context->module != NULL);
365 context->base.instr.created_functions++;
373 context->module_generation,
384 #if LLVM_VERSION_MAJOR > 11 || \
385 defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
395 if (!context->compiled)
405 #if LLVM_VERSION_MAJOR > 11
406 foreach(lc, context->handles)
412 LLVMOrcJITTargetAddress addr;
419 elog(
ERROR,
"failed to look up symbol \"%s\": %s",
433 return (
void *) (uintptr_t) addr;
435 #elif defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
436 foreach(lc, context->handles)
438 LLVMOrcTargetAddress addr;
445 return (
void *) (uintptr_t) addr;
447 #elif LLVM_VERSION_MAJOR < 5
449 LLVMOrcTargetAddress addr;
452 return (
void *) (uintptr_t) addr;
454 return (
void *) (uintptr_t) addr;
458 LLVMOrcTargetAddress addr;
463 return (
void *) (uintptr_t) addr;
467 return (
void *) (uintptr_t) addr;
483 LLVMValueRef v_srcvar;
489 elog(
ERROR,
"variable %s not in llvmjit_types.c", varname);
503 LLVMValueRef v_srcvar;
508 elog(
ERROR,
"function %s not in llvmjit_types.c", varname);
525 LLVMValueRef v_srcfn;
529 v_fn = LLVMGetNamedFunction(mod,
funcname);
538 v_fn = LLVMAddFunction(mod,
554 LLVMAttributeRef *attrs;
562 if (num_attributes == 0)
565 attrs =
palloc(
sizeof(LLVMAttributeRef) * num_attributes);
566 LLVMGetAttributesAtIndex(v_from,
index, attrs);
568 for (
int attno = 0; attno < num_attributes; attno++)
569 LLVMAddAttributeAtIndex(v_to,
index, attrs[attno]);
590 param_count = LLVMCountParams(v_from);
592 for (
int paramidx = 1; paramidx <= param_count; paramidx++)
601 LLVMBuilderRef builder,
613 if (modname != NULL && basename != NULL)
618 else if (basename != NULL)
630 LLVMValueRef v_fn_addr;
634 v_fn = LLVMGetNamedGlobal(mod,
funcname);
641 LLVMSetInitializer(v_fn, v_fn_addr);
642 LLVMSetGlobalConstant(v_fn,
true);
643 LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
644 LLVMSetUnnamedAddr(v_fn,
true);
650 v_fn = LLVMGetNamedFunction(mod,
funcname);
665 #if LLVM_VERSION_MAJOR < 17
666 LLVMPassManagerBuilderRef llvm_pmb;
667 LLVMPassManagerRef llvm_mpm;
668 LLVMPassManagerRef llvm_fpm;
670 int compile_optlevel;
673 compile_optlevel = 3;
675 compile_optlevel = 0;
682 llvm_pmb = LLVMPassManagerBuilderCreate();
683 LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
684 llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
689 LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
694 LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
697 LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
703 LLVMInitializeFunctionPassManager(llvm_fpm);
704 for (func = LLVMGetFirstFunction(context->module);
706 func = LLVMGetNextFunction(func))
707 LLVMRunFunctionPassManager(llvm_fpm, func);
708 LLVMFinalizeFunctionPassManager(llvm_fpm);
709 LLVMDisposePassManager(llvm_fpm);
715 llvm_mpm = LLVMCreatePassManager();
716 LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
720 LLVMAddAlwaysInlinerPass(llvm_mpm);
724 LLVMAddFunctionInliningPass(llvm_mpm);
725 LLVMRunPassManager(llvm_mpm, context->module);
726 LLVMDisposePassManager(llvm_mpm);
728 LLVMPassManagerBuilderDispose(llvm_pmb);
730 LLVMPassBuilderOptionsRef
options;
735 passes =
"default<O3>";
737 passes =
"default<O0>,mem2reg";
739 options = LLVMCreatePassBuilderOptions();
741 #ifdef LLVM_PASS_DEBUG
742 LLVMPassBuilderOptionsSetDebugLogging(
options, 1);
745 LLVMPassBuilderOptionsSetInlinerThreshold(
options, 512);
747 err = LLVMRunPasses(module, passes, NULL,
options);
750 elog(
ERROR,
"failed to JIT module: %s", llvm_error_message(
err));
752 LLVMDisposePassBuilderOptions(
options);
766 #if LLVM_VERSION_MAJOR > 11
767 LLVMOrcLLJITRef compile_orc;
769 LLVMOrcJITStackRef compile_orc;
793 context->module_generation);
794 LLVMWriteBitcodeToFile(context->module,
filename);
812 context->module_generation);
813 LLVMWriteBitcodeToFile(context->module,
filename);
827 #if LLVM_VERSION_MAJOR > 11
829 LLVMOrcThreadSafeModuleRef ts_module;
831 LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
833 ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
835 handle->lljit = compile_orc;
836 handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
844 context->module = NULL;
845 error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
846 handle->resource_tracker,
851 llvm_error_message(
error));
855 #elif LLVM_VERSION_MAJOR > 6
857 handle->
stack = compile_orc;
858 if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->
orc_handle, context->module,
864 #elif LLVM_VERSION_MAJOR > 4
866 LLVMSharedModuleRef smod;
868 smod = LLVMOrcMakeSharedModule(context->module);
869 handle->
stack = compile_orc;
870 if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->
orc_handle, smod,
874 LLVMOrcDisposeSharedModuleRef(smod);
878 handle->
stack = compile_orc;
879 handle->
orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
882 LLVMDisposeModule(context->module);
890 context->module = NULL;
891 context->compiled =
true;
895 context->handles =
lappend(context->handles, handle);
916 char *features = NULL;
917 LLVMTargetMachineRef opt0_tm;
918 LLVMTargetMachineRef opt3_tm;
925 LLVMInitializeNativeTarget();
926 LLVMInitializeNativeAsmPrinter();
927 LLVMInitializeNativeAsmParser();
946 #if LLVM_VERSION_MAJOR == 15
947 LLVMContextSetOpaquePointers(LLVMGetGlobalContext(),
false);
972 cpu = LLVMGetHostCPUName();
973 features = LLVMGetHostCPUFeatures();
974 elog(
DEBUG2,
"LLVMJIT detected CPU \"%s\", with features \"%s\"",
979 LLVMCodeGenLevelNone,
981 LLVMCodeModelJITDefault);
984 LLVMCodeGenLevelAggressive,
986 LLVMCodeModelJITDefault);
988 LLVMDisposeMessage(cpu);
990 LLVMDisposeMessage(features);
994 LLVMLoadLibraryPermanently(NULL);
996 #if LLVM_VERSION_MAJOR > 11
998 llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
1011 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1014 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1020 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1023 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1057 elog(
PANIC,
"LLVMJitContext in use count not 0 at exit (is %zu)",
1060 #if LLVM_VERSION_MAJOR > 11
1072 if (llvm_ts_context)
1074 LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
1075 llvm_ts_context = NULL;
1084 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
1094 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
1113 value = LLVMGetNamedFunction(mod,
name);
1130 elog(
ERROR,
"failed to extract target information, llvmjit_types.c not loaded");
1149 LLVMMemoryBufferRef
buf;
1155 if (LLVMCreateMemoryBufferWithContentsOfFile(path, &
buf, &msg))
1157 elog(
ERROR,
"LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
1164 elog(
ERROR,
"LLVMParseBitcodeInContext2 of %s failed", path);
1166 LLVMDisposeMemoryBuffer(
buf);
1208 if (strncmp(
name,
"pgextern.", strlen(
"pgextern.")) == 0)
1244 #if defined(__darwin__)
1245 if (symname[0] !=
'_')
1246 elog(
ERROR,
"expected prefixed symbol name, but got \"%s\"", symname);
1259 addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1267 elog(
WARNING,
"failed to resolve name %s", symname);
1269 return (uint64_t) addr;
1272 #if LLVM_VERSION_MAJOR > 11
1275 llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj,
void *Ctx,
1276 LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1277 LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1278 LLVMOrcCLookupSet LookupSet,
size_t LookupSetSize)
1280 #if LLVM_VERSION_MAJOR > 14
1281 LLVMOrcCSymbolMapPairs symbols =
palloc0(
sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
1283 LLVMOrcCSymbolMapPairs symbols =
palloc0(
sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1286 LLVMOrcMaterializationUnitRef mu;
1288 for (
int i = 0;
i < LookupSetSize;
i++)
1290 const char *
name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[
i].
Name);
1292 #if LLVM_VERSION_MAJOR > 12
1293 LLVMOrcRetainSymbolStringPoolEntry(LookupSet[
i].
Name);
1295 symbols[
i].Name = LookupSet[
i].Name;
1297 symbols[
i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1300 mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1301 error = LLVMOrcJITDylibDefine(JD, mu);
1302 if (
error != LLVMErrorSuccess)
1303 LLVMOrcDisposeMaterializationUnit(mu);
1320 llvm_log_jit_error(
void *ctx, LLVMErrorRef
error)
1323 llvm_error_message(
error));
1329 static LLVMOrcObjectLayerRef
1330 llvm_create_object_layer(
void *Ctx, LLVMOrcExecutionSessionRef ES,
const char *Triple)
1332 LLVMOrcObjectLayerRef objlayer =
1333 LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1335 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1338 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1340 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1344 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1347 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1349 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1360 static LLVMOrcLLJITRef
1361 llvm_create_jit_instance(LLVMTargetMachineRef
tm)
1363 LLVMOrcLLJITRef lljit;
1364 LLVMOrcJITTargetMachineBuilderRef tm_builder;
1365 LLVMOrcLLJITBuilderRef lljit_builder;
1367 LLVMOrcDefinitionGeneratorRef main_gen;
1368 LLVMOrcDefinitionGeneratorRef ref_gen;
1370 lljit_builder = LLVMOrcCreateLLJITBuilder();
1371 tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(
tm);
1372 LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1374 LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1375 llvm_create_object_layer,
1378 error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1380 elog(
ERROR,
"failed to create lljit instance: %s",
1381 llvm_error_message(
error));
1383 LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1384 llvm_log_jit_error, NULL);
1390 error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1391 LLVMOrcLLJITGetGlobalPrefix(lljit),
1394 elog(
ERROR,
"failed to create generator: %s",
1395 llvm_error_message(
error));
1396 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1402 #if LLVM_VERSION_MAJOR > 14
1403 ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
1405 ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1407 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1413 llvm_error_message(LLVMErrorRef
error)
1415 char *orig = LLVMGetErrorMessage(
error);
1418 LLVMDisposeErrorMessage(orig);
static void PGresult * res
elog(ERROR, "%s: %s", p2, msg)
void * load_external_function(const char *filename, const char *funcname, bool signalNotFound, void **filehandle)
int errmsg_internal(const char *fmt,...)
int errhidestmt(bool hide_stmt)
int errhidecontext(bool hide_ctx)
#define ereport(elevel,...)
void err(int eval, const char *fmt,...)
void fmgr_symbol(Oid functionId, char **mod, char **fn)
char pkglib_path[MAXPGPATH]
#define INSTR_TIME_SET_CURRENT(t)
#define INSTR_TIME_GET_DOUBLE(t)
#define INSTR_TIME_ACCUM_DIFF(x, y, z)
void on_proc_exit(pg_on_exit_callback function, Datum arg)
bool proc_exit_inprogress
void jit_release_context(JitContext *context)
bool jit_profiling_support
bool jit_debugging_support
Assert(fmt[strlen(fmt) - 1] !='\n')
List * lappend(List *list, void *datum)
void list_free(List *list)
LLVMTypeRef StructFunctionCallInfoData
LLVMTypeRef StructMinimalTupleData
LLVMTypeRef StructMinimalTupleTableSlot
LLVMTypeRef StructPlanState
static LLVMContextRef llvm_context
static LLVMOrcJITStackRef llvm_opt3_orc
LLVMTypeRef StructExprState
static const char * llvm_triple
LLVMTypeRef StructExprEvalStep
LLVMValueRef llvm_pg_func(LLVMModuleRef mod, const char *funcname)
static void llvm_recreate_llvm_context(void)
struct LLVMJitHandle LLVMJitHandle
LLVMTypeRef TypeParamBool
LLVMTypeRef StructMemoryContextData
static void llvm_set_target(void)
static LLVMTargetRef llvm_targetref
static bool llvm_session_initialized
LLVMTypeRef StructAggStatePerGroupData
static void llvm_create_types(void)
LLVMTypeRef llvm_pg_var_type(const char *varname)
static void ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
static void llvm_session_initialize(void)
static size_t llvm_jit_context_in_use_count
static const char * llvm_layout
LLVMValueRef llvm_function_reference(LLVMJitContext *context, LLVMBuilderRef builder, LLVMModuleRef mod, FunctionCallInfo fcinfo)
char * llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
static LLVMOrcJITStackRef llvm_opt0_orc
static void ResOwnerReleaseJitContext(Datum res)
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
LLVMTypeRef llvm_pg_var_func_type(const char *varname)
LLVMTypeRef StructTupleTableSlot
LLVMTypeRef TypeStorageBool
static LLVMTypeRef load_return_type(LLVMModuleRef mod, const char *name)
static size_t llvm_generation
static void ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
static void llvm_compile_module(LLVMJitContext *context)
LLVMTypeRef StructHeapTupleTableSlot
static LLVMModuleRef llvm_types_module
static void llvm_shutdown(int code, Datum arg)
LLVMJitContext * llvm_create_context(int jitFlags)
static const ResourceOwnerDesc jit_resowner_desc
#define LLVMJIT_LLVM_CONTEXT_REUSE_MAX
static void llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
LLVMModuleRef llvm_mutable_module(LLVMJitContext *context)
LLVMTypeRef StructAggState
LLVMTypeRef TypePGFunction
LLVMTypeRef StructTupleDescData
LLVMValueRef AttributeTemplate
static size_t llvm_llvm_context_reuse_count
LLVMTypeRef StructExprContext
static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
static uint64_t llvm_resolve_symbol(const char *name, void *ctx)
LLVMTypeRef StructHeapTupleHeaderData
LLVMTypeRef StructHeapTupleData
static void llvm_release_context(JitContext *context)
LLVMTypeRef StructNullableDatum
LLVMValueRef ExecEvalSubroutineTemplate
LLVMValueRef ExecEvalBoolSubroutineTemplate
void _PG_jit_provider_init(JitProviderCallbacks *cb)
void * llvm_get_function(LLVMJitContext *context, const char *funcname)
LLVMTypeRef StructAggStatePerTransData
void llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
void llvm_enter_fatal_on_oom(void)
bool llvm_in_fatal_on_oom(void)
void llvm_assert_in_fatal_section(void)
void llvm_reset_after_error(void)
void llvm_leave_fatal_on_oom(void)
bool llvm_compile_expr(ExprState *state)
void llvm_inline(LLVMModuleRef M)
void llvm_inline_reset_caches(void)
LLVMTypeRef LLVMGetFunctionReturnType(LLVMValueRef r)
unsigned LLVMGetAttributeCountAtIndexPG(LLVMValueRef F, uint32 Idx)
LLVMTypeRef LLVMGlobalGetValueType(LLVMValueRef g)
LLVMTypeRef LLVMGetFunctionType(LLVMValueRef r)
char * pnstrdup(const char *in, Size len)
char * pstrdup(const char *in)
void pfree(void *pointer)
MemoryContext TopMemoryContext
void * palloc0(Size size)
void * MemoryContextAllocZero(MemoryContext context, Size size)
void * MemoryContextAlloc(MemoryContext context, Size size)
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
static Datum PointerGetDatum(const void *X)
static Pointer DatumGetPointer(Datum X)
char * psprintf(const char *fmt,...)
ResourceOwner CurrentResourceOwner
void ResourceOwnerForget(ResourceOwner owner, Datum value, const ResourceOwnerDesc *kind)
void ResourceOwnerRemember(ResourceOwner owner, Datum value, const ResourceOwnerDesc *kind)
void ResourceOwnerEnlarge(ResourceOwner owner)
#define RELEASE_PRIO_JIT_CONTEXTS
@ RESOURCE_RELEASE_BEFORE_LOCKS
JitProviderCompileExprCB compile_expr
JitProviderResetAfterErrorCB reset_after_error
JitProviderReleaseContextCB release_context
LLVMOrcModuleHandle orc_handle