16 #include <llvm-c/Analysis.h> 17 #include <llvm-c/BitReader.h> 18 #include <llvm-c/BitWriter.h> 19 #include <llvm-c/Core.h> 20 #include <llvm-c/ExecutionEngine.h> 21 #if LLVM_VERSION_MAJOR > 11 22 #include <llvm-c/Orc.h> 23 #include <llvm-c/OrcEE.h> 24 #include <llvm-c/LLJIT.h> 26 #include <llvm-c/OrcBindings.h> 28 #include <llvm-c/Support.h> 29 #include <llvm-c/Target.h> 30 #include <llvm-c/Transforms/IPO.h> 31 #include <llvm-c/Transforms/PassManagerBuilder.h> 32 #include <llvm-c/Transforms/Scalar.h> 33 #if LLVM_VERSION_MAJOR > 6 34 #include <llvm-c/Transforms/Utils.h> 48 #if LLVM_VERSION_MAJOR > 11 49 LLVMOrcLLJITRef lljit;
50 LLVMOrcResourceTrackerRef resource_tracker;
100 #if LLVM_VERSION_MAJOR > 11 101 static LLVMOrcThreadSafeContextRef llvm_ts_context;
119 #if LLVM_VERSION_MAJOR > 11 120 static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef
tm);
121 static char *llvm_error_message(LLVMErrorRef
error);
148 LLVMJitContext *context;
157 sizeof(LLVMJitContext));
158 context->base.flags = jitFlags;
173 LLVMJitContext *llvm_context = (LLVMJitContext *) context;
185 if (llvm_context->module)
187 LLVMDisposeModule(llvm_context->module);
188 llvm_context->module = NULL;
191 while (llvm_context->handles !=
NIL)
198 #if LLVM_VERSION_MAJOR > 11 200 LLVMOrcExecutionSessionRef ee;
201 LLVMOrcSymbolStringPoolRef sp;
203 LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
204 LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
212 ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
213 sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
214 LLVMOrcSymbolStringPoolClearDeadEntries(sp);
237 if (!context->module)
239 context->compiled =
false;
241 context->module = LLVMModuleCreateWithName(
"pg");
246 return context->module;
257 Assert(context->module != NULL);
259 context->base.instr.created_functions++;
267 context->module_generation,
278 #if LLVM_VERSION_MAJOR > 11 || \ 279 defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN 289 if (!context->compiled)
299 #if LLVM_VERSION_MAJOR > 11 300 foreach(lc, context->handles)
306 LLVMOrcJITTargetAddress addr;
311 error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
313 elog(
ERROR,
"failed to look up symbol \"%s\": %s",
314 funcname, llvm_error_message(error));
327 return (
void *) (uintptr_t) addr;
329 #elif defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN 330 foreach(lc, context->handles)
332 LLVMOrcTargetAddress addr;
336 if (LLVMOrcGetSymbolAddressIn(handle->
stack, &addr, handle->
orc_handle, funcname))
337 elog(
ERROR,
"failed to look up symbol \"%s\"", funcname);
339 return (
void *) (uintptr_t) addr;
341 #elif LLVM_VERSION_MAJOR < 5 343 LLVMOrcTargetAddress addr;
345 if ((addr = LLVMOrcGetSymbolAddress(
llvm_opt0_orc, funcname)))
346 return (
void *) (uintptr_t) addr;
347 if ((addr = LLVMOrcGetSymbolAddress(
llvm_opt3_orc, funcname)))
348 return (
void *) (uintptr_t) addr;
352 LLVMOrcTargetAddress addr;
355 elog(
ERROR,
"failed to look up symbol \"%s\"", funcname);
357 return (
void *) (uintptr_t) addr;
359 elog(
ERROR,
"failed to look up symbol \"%s\"", funcname);
361 return (
void *) (uintptr_t) addr;
365 elog(
ERROR,
"failed to JIT: %s", funcname);
377 LLVMValueRef v_srcvar;
383 elog(
ERROR,
"variable %s not in llvmjit_types.c", varname);
386 typ = LLVMTypeOf(v_srcvar);
387 Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMPointerTypeKind);
388 typ = LLVMGetElementType(typ);
404 Assert(LLVMGetTypeKind(typ) == LLVMPointerTypeKind);
405 typ = LLVMGetElementType(typ);
406 Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMFunctionTypeKind);
421 LLVMValueRef v_srcfn;
425 v_fn = LLVMGetNamedFunction(mod, funcname);
432 elog(
ERROR,
"function %s not in llvmjit_types.c", funcname);
434 v_fn = LLVMAddFunction(mod,
436 LLVMGetElementType(LLVMTypeOf(v_srcfn)));
450 LLVMAttributeRef *attrs;
458 if (num_attributes == 0)
461 attrs =
palloc(
sizeof(LLVMAttributeRef) * num_attributes);
462 LLVMGetAttributesAtIndex(v_from, index, attrs);
464 for (
int attno = 0; attno < num_attributes; attno++)
465 LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
486 param_count = LLVMCountParams(v_from);
488 for (
int paramidx = 1; paramidx <= param_count; paramidx++)
497 LLVMBuilderRef builder,
509 if (modname != NULL && basename != NULL)
512 funcname =
psprintf(
"pgextern.%s.%s", modname, basename);
514 else if (basename != NULL)
517 funcname =
psprintf(
"%s", basename);
526 LLVMValueRef v_fn_addr;
528 funcname =
psprintf(
"pgoidextern.%u",
530 v_fn = LLVMGetNamedGlobal(mod, funcname);
532 return LLVMBuildLoad(builder, v_fn,
"");
537 LLVMSetInitializer(v_fn, v_fn_addr);
538 LLVMSetGlobalConstant(v_fn,
true);
539 LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
540 LLVMSetUnnamedAddr(v_fn,
true);
542 return LLVMBuildLoad(builder, v_fn,
"");
546 v_fn = LLVMGetNamedFunction(mod, funcname);
550 v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(
TypePGFunction));
561 LLVMPassManagerBuilderRef llvm_pmb;
562 LLVMPassManagerRef llvm_mpm;
563 LLVMPassManagerRef llvm_fpm;
565 int compile_optlevel;
568 compile_optlevel = 3;
570 compile_optlevel = 0;
577 llvm_pmb = LLVMPassManagerBuilderCreate();
578 LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
579 llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
584 LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
589 LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
592 LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
598 LLVMInitializeFunctionPassManager(llvm_fpm);
599 for (func = LLVMGetFirstFunction(context->module);
601 func = LLVMGetNextFunction(func))
602 LLVMRunFunctionPassManager(llvm_fpm, func);
603 LLVMFinalizeFunctionPassManager(llvm_fpm);
604 LLVMDisposePassManager(llvm_fpm);
610 llvm_mpm = LLVMCreatePassManager();
611 LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
615 LLVMAddAlwaysInlinerPass(llvm_mpm);
619 LLVMAddFunctionInliningPass(llvm_mpm);
620 LLVMRunPassManager(llvm_mpm, context->module);
621 LLVMDisposePassManager(llvm_mpm);
623 LLVMPassManagerBuilderDispose(llvm_pmb);
636 #if LLVM_VERSION_MAJOR > 11 637 LLVMOrcLLJITRef compile_orc;
639 LLVMOrcJITStackRef compile_orc;
663 context->module_generation);
664 LLVMWriteBitcodeToFile(context->module, filename);
680 filename =
psprintf(
"%u.%zu.optimized.bc",
682 context->module_generation);
683 LLVMWriteBitcodeToFile(context->module, filename);
697 #if LLVM_VERSION_MAJOR > 11 699 LLVMOrcThreadSafeModuleRef ts_module;
701 LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
703 ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
705 handle->lljit = compile_orc;
706 handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
714 context->module = NULL;
715 error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
716 handle->resource_tracker,
721 llvm_error_message(error));
723 handle->lljit = compile_orc;
727 #elif LLVM_VERSION_MAJOR > 6 729 handle->
stack = compile_orc;
730 if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->
orc_handle, context->module,
736 #elif LLVM_VERSION_MAJOR > 4 738 LLVMSharedModuleRef smod;
740 smod = LLVMOrcMakeSharedModule(context->module);
741 handle->
stack = compile_orc;
742 if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->
orc_handle, smod,
746 LLVMOrcDisposeSharedModuleRef(smod);
750 handle->
stack = compile_orc;
751 handle->
orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
754 LLVMDisposeModule(context->module);
762 context->module = NULL;
763 context->compiled =
true;
767 context->handles =
lappend(context->handles, handle);
788 char *features = NULL;
789 LLVMTargetMachineRef opt0_tm;
790 LLVMTargetMachineRef opt3_tm;
797 LLVMInitializeNativeTarget();
798 LLVMInitializeNativeAsmPrinter();
799 LLVMInitializeNativeAsmParser();
809 elog(
FATAL,
"failed to query triple %s\n", error);
818 cpu = LLVMGetHostCPUName();
819 features = LLVMGetHostCPUFeatures();
820 elog(
DEBUG2,
"LLVMJIT detected CPU \"%s\", with features \"%s\"",
825 LLVMCodeGenLevelNone,
827 LLVMCodeModelJITDefault);
830 LLVMCodeGenLevelAggressive,
832 LLVMCodeModelJITDefault);
834 LLVMDisposeMessage(cpu);
836 LLVMDisposeMessage(features);
840 LLVMLoadLibraryPermanently(NULL);
842 #if LLVM_VERSION_MAJOR > 11 844 llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
857 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER 860 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
866 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER 869 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
888 #if LLVM_VERSION_MAJOR > 11 902 LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
903 llvm_ts_context = NULL;
912 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF 922 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF 941 value = LLVMGetNamedFunction(mod, name);
943 elog(
ERROR,
"function %s is unknown", name);
946 typ = LLVMTypeOf(value);
949 typ = LLVMGetElementType(typ);
952 typ = LLVMGetReturnType(typ);
968 LLVMMemoryBufferRef
buf;
974 if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
976 elog(
ERROR,
"LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
983 elog(
ERROR,
"LLVMParseBitcode2 of %s failed", path);
985 LLVMDisposeMemoryBuffer(buf);
1029 if (strncmp(name,
"pgextern.", strlen(
"pgextern.")) == 0)
1035 *funcname = rindex(name,
'.');
1038 *modname =
pnstrdup(name + strlen(
"pgextern."),
1039 *funcname - name - strlen(
"pgextern.") - 1);
1042 *funcname =
pstrdup(*funcname);
1065 #if defined(__darwin__) 1066 if (symname[0] !=
'_')
1067 elog(
ERROR,
"expected prefixed symbol name, but got \"%s\"", symname);
1080 addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1088 elog(
WARNING,
"failed to resolve name %s", symname);
1090 return (uint64_t) addr;
1093 #if LLVM_VERSION_MAJOR > 11 1096 llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj,
void *Ctx,
1097 LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1098 LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1099 LLVMOrcCLookupSet LookupSet,
size_t LookupSetSize)
1101 LLVMOrcCSymbolMapPairs symbols =
palloc0(
sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1103 LLVMOrcMaterializationUnitRef mu;
1105 for (
int i = 0;
i < LookupSetSize;
i++)
1107 const char *
name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[
i].
Name);
1109 symbols[
i].Name = LookupSet[
i].Name;
1111 symbols[
i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1114 mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1115 error = LLVMOrcJITDylibDefine(JD, mu);
1116 if (error != LLVMErrorSuccess)
1117 LLVMOrcDisposeMaterializationUnit(mu);
1134 llvm_log_jit_error(
void *ctx, LLVMErrorRef
error)
1137 llvm_error_message(error));
1143 static LLVMOrcObjectLayerRef
1144 llvm_create_object_layer(
void *Ctx, LLVMOrcExecutionSessionRef ES,
const char *Triple)
1146 LLVMOrcObjectLayerRef objlayer =
1147 LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1149 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER 1152 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1154 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1158 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER 1161 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1163 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1174 static LLVMOrcLLJITRef
1175 llvm_create_jit_instance(LLVMTargetMachineRef
tm)
1177 LLVMOrcLLJITRef lljit;
1178 LLVMOrcJITTargetMachineBuilderRef tm_builder;
1179 LLVMOrcLLJITBuilderRef lljit_builder;
1181 LLVMOrcDefinitionGeneratorRef main_gen;
1182 LLVMOrcDefinitionGeneratorRef ref_gen;
1184 lljit_builder = LLVMOrcCreateLLJITBuilder();
1185 tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1186 LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1188 LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1189 llvm_create_object_layer,
1192 error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1194 elog(
ERROR,
"failed to create lljit instance: %s",
1195 llvm_error_message(error));
1197 LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1198 llvm_log_jit_error, NULL);
1204 error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1205 LLVMOrcLLJITGetGlobalPrefix(lljit),
1208 elog(
ERROR,
"failed to create generator: %s",
1209 llvm_error_message(error));
1210 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1216 ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1217 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1223 llvm_error_message(LLVMErrorRef
error)
1225 char *orig = LLVMGetErrorMessage(error);
1228 LLVMDisposeErrorMessage(orig);
LLVMTypeRef StructPGFinfoRecord
JitProviderResetAfterErrorCB reset_after_error
LLVMTypeRef StructTupleConstr
LLVMTypeRef StructFunctionCallInfoData
static void llvm_compile_module(LLVMJitContext *context)
char * pnstrdup(const char *in, Size len)
LLVMTypeRef StructHeapTupleFieldsField3
LLVMTypeRef StructMinimalTupleData
void on_proc_exit(pg_on_exit_callback function, Datum arg)
LLVMTypeRef StructExprEvalStep
#define PointerGetDatum(X)
LLVMTypeRef StructBlockId
LLVMTypeRef StructExprState
LLVMTypeRef StructHeapTupleFields
ResourceOwner CurrentResourceOwner
char * pstrdup(const char *in)
char * psprintf(const char *fmt,...)
bool jit_debugging_support
struct timeval instr_time
static void llvm_shutdown(int code, Datum arg)
LLVMTypeRef StructNullableDatum
LLVMTypeRef StructItemPointerData
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
void _PG_jit_provider_init(JitProviderCallbacks *cb)
LLVMValueRef llvm_function_reference(LLVMJitContext *context, LLVMBuilderRef builder, LLVMModuleRef mod, FunctionCallInfo fcinfo)
int errhidestmt(bool hide_stmt)
#define INSTR_TIME_ACCUM_DIFF(x, y, z)
static const char * llvm_layout
LLVMJitContext * llvm_create_context(int jitFlags)
LLVMTypeRef StructFmgrInfo
#define INSTR_TIME_GET_DOUBLE(t)
LLVMTypeRef StructHeapTupleTableSlot
LLVMTypeRef StructAggState
static LLVMTypeRef load_return_type(LLVMModuleRef mod, const char *name)
void fmgr_symbol(Oid functionId, char **mod, char **fn)
static const char * llvm_triple
LLVMTypeRef llvm_pg_var_type(const char *varname)
LLVMTypeRef StructHeapTupleData
LLVMTypeRef StructHeapTupleDataChoice
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
void pfree(void *pointer)
struct LLVMJitHandle LLVMJitHandle
static uint64_t llvm_resolve_symbol(const char *name, void *ctx)
unsigned LLVMGetAttributeCountAtIndexPG(LLVMValueRef F, uint32 Idx)
LLVMOrcModuleHandle orc_handle
char * llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
void llvm_assert_in_fatal_section(void)
LLVMTypeRef TypeStorageBool
LLVMTypeRef StructMemoryContextData
LLVMTypeRef StructExprContext
MemoryContext TopMemoryContext
static LLVMOrcJITStackRef llvm_opt3_orc
void * load_external_function(const char *filename, const char *funcname, bool signalNotFound, void **filehandle)
List * lappend(List *list, void *datum)
LLVMTypeRef TypePGFunction
LLVMTypeRef StructAggStatePerTransData
JitProviderCompileExprCB compile_expr
void ResourceOwnerRememberJIT(ResourceOwner owner, Datum handle)
void * palloc0(Size size)
static bool llvm_session_initialized
void llvm_reset_after_error(void)
LLVMTypeRef StructMinimalTupleTableSlot
static size_t llvm_generation
void * MemoryContextAllocZero(MemoryContext context, Size size)
LLVMTypeRef StructTupleTableSlot
#define ereport(elevel,...)
LLVMModuleRef llvm_mutable_module(LLVMJitContext *context)
static LLVMTargetRef llvm_targetref
int errmsg_internal(const char *fmt,...)
static void llvm_session_initialize(void)
static void llvm_release_context(JitContext *context)
#define Assert(condition)
bool proc_exit_inprogress
bool llvm_compile_expr(ExprState *state)
LLVMTypeRef StructTupleDescData
LLVMTypeRef TypeParamBool
bool jit_profiling_support
void * llvm_get_function(LLVMJitContext *context, const char *funcname)
#define INSTR_TIME_SET_CURRENT(t)
void llvm_inline(LLVMModuleRef M)
static void llvm_create_types(void)
void * MemoryContextAlloc(MemoryContext context, Size size)
LLVMModuleRef llvm_types_module
void llvm_enter_fatal_on_oom(void)
LLVMValueRef AttributeTemplate
static void llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
LLVMTypeRef StructHeapTupleHeaderData
void ResourceOwnerEnlargeJIT(ResourceOwner owner)
void llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
static LLVMOrcJITStackRef llvm_opt0_orc
JitProviderReleaseContextCB release_context
static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
char pkglib_path[MAXPGPATH]
int errhidecontext(bool hide_ctx)
List * list_delete_first(List *list)
LLVMTypeRef llvm_pg_var_func_type(const char *varname)
LLVMTypeRef StructFormPgAttribute
LLVMValueRef llvm_pg_func(LLVMModuleRef mod, const char *funcname)
LLVMTypeRef StructAggStatePerGroupData