16#include <llvm-c/Analysis.h>
17#include <llvm-c/BitReader.h>
18#include <llvm-c/BitWriter.h>
19#include <llvm-c/Core.h>
20#include <llvm-c/ExecutionEngine.h>
21#if LLVM_VERSION_MAJOR > 16
22#include <llvm-c/Transforms/PassBuilder.h>
24#include <llvm-c/Orc.h>
25#include <llvm-c/OrcEE.h>
26#include <llvm-c/LLJIT.h>
27#include <llvm-c/Support.h>
28#include <llvm-c/Target.h>
29#if LLVM_VERSION_MAJOR < 17
30#include <llvm-c/Transforms/IPO.h>
31#include <llvm-c/Transforms/PassManagerBuilder.h>
32#include <llvm-c/Transforms/Scalar.h>
33#include <llvm-c/Transforms/Utils.h>
45#define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
122 .
name =
"LLVM JIT context",
173 elog(
ERROR,
"Trying to recreate a non-existing context");
233 sizeof(LLVMJitContext));
234 context->base.flags = jitFlags;
251 LLVMJitContext *llvm_jit_context = (LLVMJitContext *)
context;
270 if (llvm_jit_context->module)
272 LLVMDisposeModule(llvm_jit_context->module);
273 llvm_jit_context->module = NULL;
276 foreach(lc, llvm_jit_context->handles)
281 LLVMOrcExecutionSessionRef ee;
282 LLVMOrcSymbolStringPoolRef sp;
293 ee = LLVMOrcLLJITGetExecutionSession(jit_handle->
lljit);
294 sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
295 LLVMOrcSymbolStringPoolClearDeadEntries(sp);
301 llvm_jit_context->handles =
NIL;
305 if (llvm_jit_context->resowner)
342 context->base.instr.created_functions++;
385 LLVMOrcJITTargetAddress addr;
392 elog(
ERROR,
"failed to look up symbol \"%s\": %s",
406 return (
void *) (uintptr_t) addr;
421 LLVMValueRef v_srcvar;
427 elog(
ERROR,
"variable %s not in llvmjit_types.c", varname);
429 typ = LLVMGlobalGetValueType(v_srcvar);
441 LLVMValueRef v_srcvar;
446 elog(
ERROR,
"function %s not in llvmjit_types.c", varname);
463 LLVMValueRef v_srcfn;
467 v_fn = LLVMGetNamedFunction(mod,
funcname);
476 v_fn = LLVMAddFunction(mod,
492 LLVMAttributeRef *attrs;
494 num_attributes = LLVMGetAttributeCountAtIndex(v_from,
index);
496 if (num_attributes == 0)
499 attrs =
palloc(
sizeof(LLVMAttributeRef) * num_attributes);
500 LLVMGetAttributesAtIndex(v_from,
index, attrs);
502 for (
int attno = 0; attno < num_attributes; attno++)
503 LLVMAddAttributeAtIndex(v_to,
index, attrs[attno]);
527 param_count = LLVMCountParams(v_from);
529 for (
int paramidx = 1; paramidx <= param_count; paramidx++)
538 LLVMBuilderRef builder,
550 if (modname != NULL && basename != NULL)
555 else if (basename != NULL)
567 LLVMValueRef v_fn_addr;
571 v_fn = LLVMGetNamedGlobal(mod,
funcname);
578 LLVMSetInitializer(v_fn, v_fn_addr);
579 LLVMSetGlobalConstant(v_fn,
true);
580 LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
581 LLVMSetUnnamedAddr(v_fn,
true);
587 v_fn = LLVMGetNamedFunction(mod,
funcname);
602#if LLVM_VERSION_MAJOR < 17
603 LLVMPassManagerBuilderRef llvm_pmb;
604 LLVMPassManagerRef llvm_mpm;
605 LLVMPassManagerRef llvm_fpm;
607 int compile_optlevel;
610 compile_optlevel = 3;
612 compile_optlevel = 0;
619 llvm_pmb = LLVMPassManagerBuilderCreate();
620 LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
621 llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
626 LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
631 LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
634 LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
640 LLVMInitializeFunctionPassManager(llvm_fpm);
641 for (func = LLVMGetFirstFunction(
context->module);
643 func = LLVMGetNextFunction(func))
644 LLVMRunFunctionPassManager(llvm_fpm, func);
645 LLVMFinalizeFunctionPassManager(llvm_fpm);
646 LLVMDisposePassManager(llvm_fpm);
652 llvm_mpm = LLVMCreatePassManager();
653 LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
657 LLVMAddAlwaysInlinerPass(llvm_mpm);
661 LLVMAddFunctionInliningPass(llvm_mpm);
662 LLVMRunPassManager(llvm_mpm,
context->module);
663 LLVMDisposePassManager(llvm_mpm);
665 LLVMPassManagerBuilderDispose(llvm_pmb);
667 LLVMPassBuilderOptionsRef
options;
672 passes =
"default<O3>";
674 passes =
"default<O0>,mem2reg";
676 options = LLVMCreatePassBuilderOptions();
678#ifdef LLVM_PASS_DEBUG
679 LLVMPassBuilderOptionsSetDebugLogging(
options, 1);
683#ifdef USE_ASSERT_CHECKING
684 LLVMPassBuilderOptionsSetVerifyEach(
options,
true);
687 LLVMPassBuilderOptionsSetInlinerThreshold(
options, 512);
689 err = LLVMRunPasses(module, passes, NULL,
options);
694 LLVMDisposePassBuilderOptions(
options);
708 LLVMOrcLLJITRef compile_orc;
766 LLVMOrcThreadSafeModuleRef ts_module;
768 LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
772 handle->
lljit = compile_orc;
782 error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
823 char *features = NULL;
824 LLVMTargetMachineRef opt0_tm;
825 LLVMTargetMachineRef opt3_tm;
832 LLVMInitializeNativeTarget();
833 LLVMInitializeNativeAsmPrinter();
834 LLVMInitializeNativeAsmParser();
866 cpu = LLVMGetHostCPUName();
867 features = LLVMGetHostCPUFeatures();
868 elog(
DEBUG2,
"LLVMJIT detected CPU \"%s\", with features \"%s\"",
873 LLVMCodeGenLevelNone,
875 LLVMCodeModelJITDefault);
878 LLVMCodeGenLevelAggressive,
880 LLVMCodeModelJITDefault);
882 LLVMDisposeMessage(cpu);
884 LLVMDisposeMessage(features);
888 LLVMLoadLibraryPermanently(NULL);
925 elog(
PANIC,
"LLVMJitContext in use count not 0 at exit (is %zu)",
972 elog(
ERROR,
"failed to extract target information, llvmjit_types.c not loaded");
991 LLVMMemoryBufferRef
buf;
997 if (LLVMCreateMemoryBufferWithContentsOfFile(path, &
buf, &msg))
999 elog(
ERROR,
"LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
1006 elog(
ERROR,
"LLVMParseBitcodeInContext2 of %s failed", path);
1008 LLVMDisposeMemoryBuffer(
buf);
1050 if (strncmp(
name,
"pgextern.", strlen(
"pgextern.")) == 0)
1086#if defined(__darwin__)
1087 if (symname[0] !=
'_')
1088 elog(
ERROR,
"expected prefixed symbol name, but got \"%s\"", symname);
1101 addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1109 elog(
WARNING,
"failed to resolve name %s", symname);
1111 return (uint64_t) addr;
1116 LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1117 LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1118 LLVMOrcCLookupSet LookupSet,
size_t LookupSetSize)
1120#if LLVM_VERSION_MAJOR > 14
1121 LLVMOrcCSymbolMapPairs symbols =
palloc0(
sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
1123 LLVMOrcCSymbolMapPairs symbols =
palloc0(
sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1126 LLVMOrcMaterializationUnitRef mu;
1128 for (
int i = 0;
i < LookupSetSize;
i++)
1130 const char *
name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[
i].
Name);
1132 LLVMOrcRetainSymbolStringPoolEntry(LookupSet[
i].
Name);
1133 symbols[
i].Name = LookupSet[
i].Name;
1135 symbols[
i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1138 mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1139 error = LLVMOrcJITDylibDefine(JD, mu);
1140 if (
error != LLVMErrorSuccess)
1141 LLVMOrcDisposeMaterializationUnit(mu);
1167static LLVMOrcObjectLayerRef
1170#ifdef USE_LLVM_BACKPORT_SECTION_MEMORY_MANAGER
1171 LLVMOrcObjectLayerRef objlayer =
1172 LLVMOrcCreateRTDyldObjectLinkingLayerWithSafeSectionMemoryManager(ES);
1174 LLVMOrcObjectLayerRef objlayer =
1175 LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1179#if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1182 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1184 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1188#if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1191 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1193 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1204static LLVMOrcLLJITRef
1207 LLVMOrcLLJITRef lljit;
1208 LLVMOrcJITTargetMachineBuilderRef tm_builder;
1209 LLVMOrcLLJITBuilderRef lljit_builder;
1211 LLVMOrcDefinitionGeneratorRef main_gen;
1212 LLVMOrcDefinitionGeneratorRef ref_gen;
1214 lljit_builder = LLVMOrcCreateLLJITBuilder();
1215 tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(
tm);
1216 LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1218 LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1222 error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1224 elog(
ERROR,
"failed to create lljit instance: %s",
1227 LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1234 error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1235 LLVMOrcLLJITGetGlobalPrefix(lljit),
1238 elog(
ERROR,
"failed to create generator: %s",
1240 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1246#if LLVM_VERSION_MAJOR > 14
1251 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1259 char *orig = LLVMGetErrorMessage(
error);
1262 LLVMDisposeErrorMessage(orig);
#define Assert(condition)
static void PGresult * res
void * load_external_function(const char *filename, const char *funcname, bool signalNotFound, void **filehandle)
int errmsg_internal(const char *fmt,...)
int errhidestmt(bool hide_stmt)
int errhidecontext(bool hide_ctx)
#define ereport(elevel,...)
void err(int eval, const char *fmt,...)
void fmgr_symbol(Oid functionId, char **mod, char **fn)
char pkglib_path[MAXPGPATH]
#define INSTR_TIME_SET_CURRENT(t)
#define INSTR_TIME_GET_DOUBLE(t)
#define INSTR_TIME_ACCUM_DIFF(x, y, z)
void on_proc_exit(pg_on_exit_callback function, Datum arg)
bool proc_exit_inprogress
void jit_release_context(JitContext *context)
bool jit_profiling_support
bool jit_debugging_support
List * lappend(List *list, void *datum)
void list_free(List *list)
LLVMTypeRef StructFunctionCallInfoData
LLVMTypeRef StructMinimalTupleData
LLVMTypeRef StructMinimalTupleTableSlot
LLVMTypeRef StructPlanState
static LLVMContextRef llvm_context
LLVMJitContext * llvm_create_context(int jitFlags)
LLVMTypeRef StructExprState
static LLVMOrcLLJITRef llvm_opt3_orc
static const char * llvm_triple
LLVMTypeRef StructExprEvalStep
LLVMValueRef llvm_pg_func(LLVMModuleRef mod, const char *funcname)
static void llvm_recreate_llvm_context(void)
struct LLVMJitHandle LLVMJitHandle
LLVMTypeRef TypeParamBool
LLVMTypeRef StructMemoryContextData
static void llvm_set_target(void)
static LLVMTargetRef llvm_targetref
static bool llvm_session_initialized
LLVMTypeRef StructAggStatePerGroupData
static void llvm_create_types(void)
LLVMTypeRef llvm_pg_var_type(const char *varname)
static void ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
static void llvm_session_initialize(void)
static size_t llvm_jit_context_in_use_count
static const char * llvm_layout
LLVMValueRef llvm_function_reference(LLVMJitContext *context, LLVMBuilderRef builder, LLVMModuleRef mod, FunctionCallInfo fcinfo)
char * llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
static void ResOwnerReleaseJitContext(Datum res)
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
LLVMTypeRef llvm_pg_var_func_type(const char *varname)
LLVMTypeRef StructTupleTableSlot
LLVMTypeRef TypeStorageBool
static LLVMTypeRef load_return_type(LLVMModuleRef mod, const char *name)
static size_t llvm_generation
static LLVMOrcLLJITRef llvm_opt0_orc
static LLVMOrcObjectLayerRef llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
static void ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
static void llvm_compile_module(LLVMJitContext *context)
LLVMTypeRef StructHeapTupleTableSlot
static char * llvm_error_message(LLVMErrorRef error)
static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm)
static LLVMModuleRef llvm_types_module
static void llvm_shutdown(int code, Datum arg)
static LLVMOrcThreadSafeContextRef llvm_ts_context
static const ResourceOwnerDesc jit_resowner_desc
#define LLVMJIT_LLVM_CONTEXT_REUSE_MAX
static LLVMErrorRef llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx, LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind, LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags, LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
static void llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
LLVMModuleRef llvm_mutable_module(LLVMJitContext *context)
LLVMTypeRef StructAggState
LLVMTypeRef TypePGFunction
LLVMTypeRef StructTupleDescData
LLVMValueRef AttributeTemplate
static size_t llvm_llvm_context_reuse_count
LLVMTypeRef StructExprContext
static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
static uint64_t llvm_resolve_symbol(const char *name, void *ctx)
LLVMTypeRef StructHeapTupleHeaderData
static void llvm_log_jit_error(void *ctx, LLVMErrorRef error)
LLVMTypeRef StructHeapTupleData
static void llvm_release_context(JitContext *context)
void * llvm_get_function(LLVMJitContext *context, const char *funcname)
LLVMTypeRef StructNullableDatum
LLVMValueRef ExecEvalSubroutineTemplate
LLVMValueRef ExecEvalBoolSubroutineTemplate
void _PG_jit_provider_init(JitProviderCallbacks *cb)
LLVMTypeRef StructAggStatePerTransData
void llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
void llvm_enter_fatal_on_oom(void)
bool llvm_in_fatal_on_oom(void)
void llvm_assert_in_fatal_section(void)
void llvm_reset_after_error(void)
void llvm_leave_fatal_on_oom(void)
bool llvm_compile_expr(ExprState *state)
void llvm_inline(LLVMModuleRef M)
void llvm_inline_reset_caches(void)
LLVMTypeRef LLVMGetFunctionReturnType(LLVMValueRef r)
LLVMTypeRef LLVMGetFunctionType(LLVMValueRef r)
void * MemoryContextAlloc(MemoryContext context, Size size)
void * MemoryContextAllocZero(MemoryContext context, Size size)
char * pstrdup(const char *in)
void pfree(void *pointer)
void * palloc0(Size size)
MemoryContext TopMemoryContext
char * pnstrdup(const char *in, Size len)
static Datum PointerGetDatum(const void *X)
static Pointer DatumGetPointer(Datum X)
char * psprintf(const char *fmt,...)
MemoryContextSwitchTo(old_ctx)
ResourceOwner CurrentResourceOwner
void ResourceOwnerForget(ResourceOwner owner, Datum value, const ResourceOwnerDesc *kind)
void ResourceOwnerRemember(ResourceOwner owner, Datum value, const ResourceOwnerDesc *kind)
void ResourceOwnerEnlarge(ResourceOwner owner)
#define RELEASE_PRIO_JIT_CONTEXTS
@ RESOURCE_RELEASE_BEFORE_LOCKS
JitProviderCompileExprCB compile_expr
JitProviderResetAfterErrorCB reset_after_error
JitProviderReleaseContextCB release_context
LLVMOrcResourceTrackerRef resource_tracker