llvmjit.c File Reference

#include "postgres.h"
#include <llvm-c/Analysis.h>
#include <llvm-c/BitReader.h>
#include <llvm-c/BitWriter.h>
#include <llvm-c/Core.h>
#include <llvm-c/ExecutionEngine.h>
#include <llvm-c/OrcBindings.h>
#include <llvm-c/Support.h>
#include <llvm-c/Target.h>
#include <llvm-c/Transforms/IPO.h>
#include <llvm-c/Transforms/PassManagerBuilder.h>
#include <llvm-c/Transforms/Scalar.h>
#include "jit/llvmjit.h"
#include "jit/llvmjit_emit.h"
#include "miscadmin.h"
#include "portability/instr_time.h"
#include "storage/ipc.h"
#include "utils/memutils.h"
#include "utils/resowner_private.h"

Include dependency graph for llvmjit.c:

Go to the source code of this file.

Data Structures
struct	LLVMJitHandle

Typedefs
typedef struct LLVMJitHandle	LLVMJitHandle

Functions
static void	llvm_release_context (JitContext *context)
static void	llvm_session_initialize (void)
static void	llvm_shutdown (int code, Datum arg)
static void	llvm_compile_module (LLVMJitContext *context)
static void	llvm_optimize_module (LLVMJitContext *context, LLVMModuleRef module)
static void	llvm_create_types (void)
static uint64_t	llvm_resolve_symbol (const char *name, void *ctx)
void	_PG_jit_provider_init (JitProviderCallbacks *cb)
LLVMJitContext *	llvm_create_context (int jitFlags)
LLVMModuleRef	llvm_mutable_module (LLVMJitContext *context)
char *	llvm_expand_funcname (struct LLVMJitContext *context, const char *basename)
void *	llvm_get_function (LLVMJitContext *context, const char *funcname)
LLVMTypeRef	llvm_pg_var_type (const char *varname)
LLVMTypeRef	llvm_pg_var_func_type (const char *varname)
LLVMValueRef	llvm_pg_func (LLVMModuleRef mod, const char *funcname)
static void	llvm_copy_attributes_at_index (LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
void	llvm_copy_attributes (LLVMValueRef v_from, LLVMValueRef v_to)
LLVMValueRef	llvm_function_reference (LLVMJitContext *context, LLVMBuilderRef builder, LLVMModuleRef mod, FunctionCallInfo fcinfo)
static LLVMTypeRef	load_return_type (LLVMModuleRef mod, const char *name)
void	llvm_split_symbol_name (const char *name, char **modname, char **funcname)

Variables
LLVMTypeRef	TypeSizeT
LLVMTypeRef	TypeParamBool
LLVMTypeRef	TypeStorageBool
LLVMTypeRef	TypePGFunction
LLVMTypeRef	StructNullableDatum
LLVMTypeRef	StructHeapTupleFieldsField3
LLVMTypeRef	StructHeapTupleFields
LLVMTypeRef	StructHeapTupleHeaderData
LLVMTypeRef	StructHeapTupleDataChoice
LLVMTypeRef	StructHeapTupleData
LLVMTypeRef	StructMinimalTupleData
LLVMTypeRef	StructItemPointerData
LLVMTypeRef	StructBlockId
LLVMTypeRef	StructFormPgAttribute
LLVMTypeRef	StructTupleConstr
LLVMTypeRef	StructTupleDescData
LLVMTypeRef	StructTupleTableSlot
LLVMTypeRef	StructHeapTupleTableSlot
LLVMTypeRef	StructMinimalTupleTableSlot
LLVMTypeRef	StructMemoryContextData
LLVMTypeRef	StructPGFinfoRecord
LLVMTypeRef	StructFmgrInfo
LLVMTypeRef	StructFunctionCallInfoData
LLVMTypeRef	StructExprContext
LLVMTypeRef	StructExprEvalStep
LLVMTypeRef	StructExprState
LLVMTypeRef	StructAggState
LLVMTypeRef	StructAggStatePerGroupData
LLVMTypeRef	StructAggStatePerTransData
LLVMValueRef	AttributeTemplate
LLVMModuleRef	llvm_types_module = NULL
static bool	llvm_session_initialized = false
static size_t	llvm_generation = 0
static const char *	llvm_triple = NULL
static const char *	llvm_layout = NULL
static LLVMTargetRef	llvm_targetref
static LLVMOrcJITStackRef	llvm_opt0_orc
static LLVMOrcJITStackRef	llvm_opt3_orc
	PG_MODULE_MAGIC

Typedef Documentation

LLVMJitHandle

typedef struct LLVMJitHandle LLVMJitHandle

Function Documentation

_PG_jit_provider_init()

void _PG_jit_provider_init

(

JitProviderCallbacks *

cb

)

Definition at line 131 of file llvmjit.c.

{
    cb->reset_after_error = llvm_reset_after_error;
    cb->release_context = llvm_release_context;
    cb->compile_expr = llvm_compile_expr;
}

References JitProviderCallbacks::compile_expr, llvm_compile_expr(), llvm_release_context(), llvm_reset_after_error(), JitProviderCallbacks::release_context, and JitProviderCallbacks::reset_after_error.

llvm_compile_module()

static void llvm_compile_module ( LLVMJitContext *  context )

static

Definition at line 630 of file llvmjit.c.

{
    LLVMJitHandle *handle;
    MemoryContext oldcontext;
    instr_time  starttime;
    instr_time  endtime;
#if LLVM_VERSION_MAJOR > 11
    LLVMOrcLLJITRef compile_orc;
#else
    LLVMOrcJITStackRef compile_orc;
#endif
 
    if (context->base.flags & PGJIT_OPT3)
        compile_orc = llvm_opt3_orc;
    else
        compile_orc = llvm_opt0_orc;
 
    /* perform inlining */
    if (context->base.flags & PGJIT_INLINE)
    {
        INSTR_TIME_SET_CURRENT(starttime);
        llvm_inline(context->module);
        INSTR_TIME_SET_CURRENT(endtime);
        INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
                              endtime, starttime);
    }
 
    if (jit_dump_bitcode)
    {
        char       *filename;
 
        filename = psprintf("%d.%zu.bc",
                            MyProcPid,
                            context->module_generation);
        LLVMWriteBitcodeToFile(context->module, filename);
        pfree(filename);
    }
 
 
    /* optimize according to the chosen optimization settings */
    INSTR_TIME_SET_CURRENT(starttime);
    llvm_optimize_module(context, context->module);
    INSTR_TIME_SET_CURRENT(endtime);
    INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
                          endtime, starttime);
 
    if (jit_dump_bitcode)
    {
        char       *filename;
 
        filename = psprintf("%d.%zu.optimized.bc",
                            MyProcPid,
                            context->module_generation);
        LLVMWriteBitcodeToFile(context->module, filename);
        pfree(filename);
    }
 
    handle = (LLVMJitHandle *)
        MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
 
    /*
     * Emit the code. Note that this can, depending on the optimization
     * settings, take noticeable resources as code emission executes low-level
     * instruction combining/selection passes etc. Without optimization a
     * faster instruction selection mechanism is used.
     */
    INSTR_TIME_SET_CURRENT(starttime);
#if LLVM_VERSION_MAJOR > 11
    {
        LLVMOrcThreadSafeModuleRef ts_module;
        LLVMErrorRef error;
        LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
 
        ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
 
        handle->lljit = compile_orc;
        handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
 
        /*
         * NB: This doesn't actually emit code. That happens lazily the first
         * time a symbol defined in the module is requested. Due to that
         * llvm_get_function() also accounts for emission time.
         */
 
        context->module = NULL; /* will be owned by LLJIT */
        error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
                                                  handle->resource_tracker,
                                                  ts_module);
 
        if (error)
            elog(ERROR, "failed to JIT module: %s",
                 llvm_error_message(error));
 
        handle->lljit = compile_orc;
 
        /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
    }
#elif LLVM_VERSION_MAJOR > 6
    {
        handle->stack = compile_orc;
        if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
                                        llvm_resolve_symbol, NULL))
            elog(ERROR, "failed to JIT module");
 
        /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
    }
#elif LLVM_VERSION_MAJOR > 4
    {
        LLVMSharedModuleRef smod;
 
        smod = LLVMOrcMakeSharedModule(context->module);
        handle->stack = compile_orc;
        if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, smod,
                                        llvm_resolve_symbol, NULL))
            elog(ERROR, "failed to JIT module");
 
        LLVMOrcDisposeSharedModuleRef(smod);
    }
#else                           /* LLVM 4.0 and 3.9 */
    {
        handle->stack = compile_orc;
        handle->orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
                                                         llvm_resolve_symbol, NULL);
 
        LLVMDisposeModule(context->module);
    }
#endif
 
    INSTR_TIME_SET_CURRENT(endtime);
    INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
                          endtime, starttime);
 
    context->module = NULL;
    context->compiled = true;
 
    /* remember emitted code for cleanup and lookups */
    oldcontext = MemoryContextSwitchTo(TopMemoryContext);
    context->handles = lappend(context->handles, handle);
    MemoryContextSwitchTo(oldcontext);
 
    ereport(DEBUG1,
            (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
                             INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
                             INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
                             INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
             errhidestmt(true),
             errhidecontext(true)));
}

References DEBUG1, elog(), ereport, errhidecontext(), errhidestmt(), errmsg_internal(), ERROR, error(), filename, INSTR_TIME_ACCUM_DIFF, INSTR_TIME_GET_DOUBLE, INSTR_TIME_SET_CURRENT, jit_dump_bitcode, lappend(), llvm_inline(), llvm_opt0_orc, llvm_opt3_orc, llvm_optimize_module(), llvm_resolve_symbol(), MemoryContextAlloc(), MemoryContextSwitchTo(), MyProcPid, LLVMJitHandle::orc_handle, pfree(), PGJIT_INLINE, PGJIT_OPT3, psprintf(), LLVMJitHandle::stack, and TopMemoryContext.

Referenced by llvm_get_function().

llvm_copy_attributes()

void llvm_copy_attributes	(	LLVMValueRef	v_from,
		LLVMValueRef	v_to
	)

Definition at line 475 of file llvmjit.c.

{
    uint32      param_count;
 
    /* copy function attributes */
    llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
 
    /* and the return value attributes */
    llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
 
    /* and each function parameter's attribute */
    param_count = LLVMCountParams(v_from);
 
    for (int paramidx = 1; paramidx <= param_count; paramidx++)
        llvm_copy_attributes_at_index(v_from, v_to, paramidx);
}

References llvm_copy_attributes_at_index().

Referenced by llvm_pg_func(), and slot_compile_deform().

llvm_copy_attributes_at_index()

static void llvm_copy_attributes_at_index ( LLVMValueRef  v_from, LLVMValueRef  v_to, uint32  index  )

static

Definition at line 447 of file llvmjit.c.

{
    int         num_attributes;
    LLVMAttributeRef *attrs;
 
    num_attributes = LLVMGetAttributeCountAtIndexPG(v_from, index);
 
    /*
     * Not just for efficiency: LLVM <= 3.9 crashes when
     * LLVMGetAttributesAtIndex() is called for an index with 0 attributes.
     */
    if (num_attributes == 0)
        return;
 
    attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
    LLVMGetAttributesAtIndex(v_from, index, attrs);
 
    for (int attno = 0; attno < num_attributes; attno++)
        LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
 
    pfree(attrs);
}

References LLVMGetAttributeCountAtIndexPG(), palloc(), and pfree().

Referenced by llvm_copy_attributes().

llvm_create_context()

LLVMJitContext* llvm_create_context

(

int

jitFlags

)

Definition at line 146 of file llvmjit.c.

{
    LLVMJitContext *context;
 
    llvm_assert_in_fatal_section();
 
    llvm_session_initialize();
 
    ResourceOwnerEnlargeJIT(CurrentResourceOwner);
 
    context = MemoryContextAllocZero(TopMemoryContext,
                                     sizeof(LLVMJitContext));
    context->base.flags = jitFlags;
 
    /* ensure cleanup */
    context->base.resowner = CurrentResourceOwner;
    ResourceOwnerRememberJIT(CurrentResourceOwner, PointerGetDatum(context));
 
    return context;
}

References CurrentResourceOwner, llvm_assert_in_fatal_section(), llvm_session_initialize(), MemoryContextAllocZero(), PointerGetDatum(), ResourceOwnerEnlargeJIT(), ResourceOwnerRememberJIT(), and TopMemoryContext.

Referenced by llvm_compile_expr().

llvm_create_types()

static void llvm_create_types ( void  )

static

Definition at line 989 of file llvmjit.c.

{
    char        path[MAXPGPATH];
    LLVMMemoryBufferRef buf;
    char       *msg;
 
    snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
 
    /* open file */
    if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
    {
        elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
             path, msg);
    }
 
    /* eagerly load contents, going to need it all */
    if (LLVMParseBitcode2(buf, &llvm_types_module))
    {
        elog(ERROR, "LLVMParseBitcode2 of %s failed", path);
    }
    LLVMDisposeMemoryBuffer(buf);
 
    /*
     * Load triple & layout from clang emitted file so we're guaranteed to be
     * compatible.
     */
    llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
    llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
 
    TypeSizeT = llvm_pg_var_type("TypeSizeT");
    TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
    TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
    TypePGFunction = llvm_pg_var_type("TypePGFunction");
    StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
    StructExprContext = llvm_pg_var_type("StructExprContext");
    StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
    StructExprState = llvm_pg_var_type("StructExprState");
    StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
    StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
    StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
    StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
    StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
    StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
    StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
    StructAggState = llvm_pg_var_type("StructAggState");
    StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
    StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
 
    AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
}

References AttributeTemplate, buf, elog(), ERROR, llvm_layout, llvm_pg_var_type(), llvm_triple, llvm_types_module, load_return_type(), MAXPGPATH, pkglib_path, pstrdup(), snprintf, StructAggState, StructAggStatePerGroupData, StructAggStatePerTransData, StructExprContext, StructExprEvalStep, StructExprState, StructFunctionCallInfoData, StructHeapTupleData, StructHeapTupleTableSlot, StructMemoryContextData, StructMinimalTupleTableSlot, StructNullableDatum, StructTupleDescData, StructTupleTableSlot, TypeParamBool, TypePGFunction, TypeSizeT, and TypeStorageBool.

Referenced by llvm_session_initialize().

llvm_expand_funcname()

char* llvm_expand_funcname	(	struct LLVMJitContext *	context,
		const char *	basename
	)

Definition at line 255 of file llvmjit.c.

{
    Assert(context->module != NULL);
 
    context->base.instr.created_functions++;
 
    /*
     * Previously we used dots to separate, but turns out some tools, e.g.
     * GDB, don't like that and truncate name.
     */
    return psprintf("%s_%zu_%d",
                    basename,
                    context->module_generation,
                    context->counter++);
}

References Assert(), and psprintf().

Referenced by slot_compile_deform().

llvm_function_reference()

LLVMValueRef llvm_function_reference	(	LLVMJitContext *	context,
		LLVMBuilderRef	builder,
		LLVMModuleRef	mod,
		FunctionCallInfo	fcinfo
	)

Definition at line 496 of file llvmjit.c.

{
    char       *modname;
    char       *basename;
    char       *funcname;
 
    LLVMValueRef v_fn;
 
    fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
 
    if (modname != NULL && basename != NULL)
    {
        /* external function in loadable library */
        funcname = psprintf("pgextern.%s.%s", modname, basename);
    }
    else if (basename != NULL)
    {
        /* internal function */
        funcname = pstrdup(basename);
    }
    else
    {
        /*
         * Function we don't know to handle, return pointer. We do so by
         * creating a global constant containing a pointer to the function.
         * Makes IR more readable.
         */
        LLVMValueRef v_fn_addr;
 
        funcname = psprintf("pgoidextern.%u",
                            fcinfo->flinfo->fn_oid);
        v_fn = LLVMGetNamedGlobal(mod, funcname);
        if (v_fn != 0)
            return LLVMBuildLoad(builder, v_fn, "");
 
        v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
 
        v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
        LLVMSetInitializer(v_fn, v_fn_addr);
        LLVMSetGlobalConstant(v_fn, true);
        LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
        LLVMSetUnnamedAddr(v_fn, true);
 
        return LLVMBuildLoad(builder, v_fn, "");
    }
 
    /* check if function already has been added */
    v_fn = LLVMGetNamedFunction(mod, funcname);
    if (v_fn != 0)
        return v_fn;
 
    v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction));
 
    return v_fn;
}

References FunctionCallInfoBaseData::flinfo, fmgr_symbol(), FmgrInfo::fn_addr, FmgrInfo::fn_oid, funcname, psprintf(), pstrdup(), and TypePGFunction.

Referenced by BuildV1Call().

llvm_get_function()

void* llvm_get_function	(	LLVMJitContext *	context,
		const char *	funcname
	)

Definition at line 276 of file llvmjit.c.

{
#if LLVM_VERSION_MAJOR > 11 || \
    defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
    ListCell   *lc;
#endif
 
    llvm_assert_in_fatal_section();
 
    /*
     * If there is a pending / not emitted module, compile and emit now.
     * Otherwise we might not find the [correct] function.
     */
    if (!context->compiled)
    {
        llvm_compile_module(context);
    }
 
    /*
     * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
     * to mangle here.
     */
 
#if LLVM_VERSION_MAJOR > 11
    foreach(lc, context->handles)
    {
        LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
        instr_time  starttime;
        instr_time  endtime;
        LLVMErrorRef error;
        LLVMOrcJITTargetAddress addr;
 
        INSTR_TIME_SET_CURRENT(starttime);
 
        addr = 0;
        error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
        if (error)
            elog(ERROR, "failed to look up symbol \"%s\": %s",
                 funcname, llvm_error_message(error));
 
        /*
         * LLJIT only actually emits code the first time a symbol is
         * referenced. Thus add lookup time to emission time. That's counting
         * a bit more than with older LLVM versions, but unlikely to ever
         * matter.
         */
        INSTR_TIME_SET_CURRENT(endtime);
        INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
                              endtime, starttime);
 
        if (addr)
            return (void *) (uintptr_t) addr;
    }
#elif defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
    foreach(lc, context->handles)
    {
        LLVMOrcTargetAddress addr;
        LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
 
        addr = 0;
        if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
            elog(ERROR, "failed to look up symbol \"%s\"", funcname);
        if (addr)
            return (void *) (uintptr_t) addr;
    }
#elif LLVM_VERSION_MAJOR < 5
    {
        LLVMOrcTargetAddress addr;
 
        if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
            return (void *) (uintptr_t) addr;
        if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
            return (void *) (uintptr_t) addr;
    }
#else
    {
        LLVMOrcTargetAddress addr;
 
        if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
            elog(ERROR, "failed to look up symbol \"%s\"", funcname);
        if (addr)
            return (void *) (uintptr_t) addr;
        if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
            elog(ERROR, "failed to look up symbol \"%s\"", funcname);
        if (addr)
            return (void *) (uintptr_t) addr;
    }
#endif
 
    elog(ERROR, "failed to JIT: %s", funcname);
 
    return NULL;
}

References elog(), ERROR, error(), funcname, INSTR_TIME_ACCUM_DIFF, INSTR_TIME_SET_CURRENT, lfirst, llvm_assert_in_fatal_section(), llvm_compile_module(), llvm_opt0_orc, llvm_opt3_orc, LLVMJitHandle::orc_handle, and LLVMJitHandle::stack.

Referenced by ExecRunCompiledExpr().

llvm_mutable_module()

LLVMModuleRef llvm_mutable_module

(

LLVMJitContext *

context

)

Definition at line 230 of file llvmjit.c.

{
    llvm_assert_in_fatal_section();
 
    /*
     * If there's no in-progress module, create a new one.
     */
    if (!context->module)
    {
        context->compiled = false;
        context->module_generation = llvm_generation++;
        context->module = LLVMModuleCreateWithName("pg");
        LLVMSetTarget(context->module, llvm_triple);
        LLVMSetDataLayout(context->module, llvm_layout);
    }
 
    return context->module;
}

References llvm_assert_in_fatal_section(), llvm_generation, llvm_layout, and llvm_triple.

Referenced by slot_compile_deform().

llvm_optimize_module()

static void llvm_optimize_module ( LLVMJitContext *  context, LLVMModuleRef  module  )

static

Definition at line 559 of file llvmjit.c.

{
    LLVMPassManagerBuilderRef llvm_pmb;
    LLVMPassManagerRef llvm_mpm;
    LLVMPassManagerRef llvm_fpm;
    LLVMValueRef func;
    int         compile_optlevel;
 
    if (context->base.flags & PGJIT_OPT3)
        compile_optlevel = 3;
    else
        compile_optlevel = 0;
 
    /*
     * Have to create a new pass manager builder every pass through, as the
     * inliner has some per-builder state. Otherwise one ends up only inlining
     * a function the first time though.
     */
    llvm_pmb = LLVMPassManagerBuilderCreate();
    LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
    llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
 
    if (context->base.flags & PGJIT_OPT3)
    {
        /* TODO: Unscientifically determined threshold */
        LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
    }
    else
    {
        /* we rely on mem2reg heavily, so emit even in the O0 case */
        LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
    }
 
    LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
 
    /*
     * Do function level optimization. This could be moved to the point where
     * functions are emitted, to reduce memory usage a bit.
     */
    LLVMInitializeFunctionPassManager(llvm_fpm);
    for (func = LLVMGetFirstFunction(context->module);
         func != NULL;
         func = LLVMGetNextFunction(func))
        LLVMRunFunctionPassManager(llvm_fpm, func);
    LLVMFinalizeFunctionPassManager(llvm_fpm);
    LLVMDisposePassManager(llvm_fpm);
 
    /*
     * Perform module level optimization. We do so even in the non-optimized
     * case, so always-inline functions etc get inlined. It's cheap enough.
     */
    llvm_mpm = LLVMCreatePassManager();
    LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
                                                    llvm_mpm);
    /* always use always-inliner pass */
    if (!(context->base.flags & PGJIT_OPT3))
        LLVMAddAlwaysInlinerPass(llvm_mpm);
    /* if doing inlining, but no expensive optimization, add inlining pass */
    if (context->base.flags & PGJIT_INLINE
        && !(context->base.flags & PGJIT_OPT3))
        LLVMAddFunctionInliningPass(llvm_mpm);
    LLVMRunPassManager(llvm_mpm, context->module);
    LLVMDisposePassManager(llvm_mpm);
 
    LLVMPassManagerBuilderDispose(llvm_pmb);
}

References PGJIT_INLINE, and PGJIT_OPT3.

Referenced by llvm_compile_module().

llvm_pg_func()

LLVMValueRef llvm_pg_func	(	LLVMModuleRef	mod,
		const char *	funcname
	)

Definition at line 419 of file llvmjit.c.

{
    LLVMValueRef v_srcfn;
    LLVMValueRef v_fn;
 
    /* don't repeatedly add function */
    v_fn = LLVMGetNamedFunction(mod, funcname);
    if (v_fn)
        return v_fn;
 
    v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
 
    if (!v_srcfn)
        elog(ERROR, "function %s not in llvmjit_types.c", funcname);
 
    v_fn = LLVMAddFunction(mod,
                           funcname,
                           LLVMGetElementType(LLVMTypeOf(v_srcfn)));
    llvm_copy_attributes(v_srcfn, v_fn);
 
    return v_fn;
}

References elog(), ERROR, funcname, llvm_copy_attributes(), and llvm_types_module.

Referenced by build_EvalXFuncInt(), and slot_compile_deform().

llvm_pg_var_func_type()

LLVMTypeRef llvm_pg_var_func_type

(

const char *

varname

)

Definition at line 399 of file llvmjit.c.

{
    LLVMTypeRef typ = llvm_pg_var_type(varname);
 
    /* look at the contained type */
    Assert(LLVMGetTypeKind(typ) == LLVMPointerTypeKind);
    typ = LLVMGetElementType(typ);
    Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMFunctionTypeKind);
 
    return typ;
}

References Assert(), and llvm_pg_var_type().

llvm_pg_var_type()

LLVMTypeRef llvm_pg_var_type

(

const char *

varname

)

Definition at line 375 of file llvmjit.c.

{
    LLVMValueRef v_srcvar;
    LLVMTypeRef typ;
 
    /* this'll return a *pointer* to the global */
    v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
    if (!v_srcvar)
        elog(ERROR, "variable %s not in llvmjit_types.c", varname);
 
    /* look at the contained type */
    typ = LLVMTypeOf(v_srcvar);
    Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMPointerTypeKind);
    typ = LLVMGetElementType(typ);
    Assert(typ != NULL);
 
    return typ;
}

References Assert(), elog(), ERROR, and llvm_types_module.

Referenced by llvm_create_types(), and llvm_pg_var_func_type().

llvm_release_context()

static void llvm_release_context ( JitContext *  context )

static

Definition at line 171 of file llvmjit.c.

{
    LLVMJitContext *llvm_context = (LLVMJitContext *) context;
    ListCell   *lc;
 
    /*
     * When this backend is exiting, don't clean up LLVM. As an error might
     * have occurred from within LLVM, we do not want to risk reentering. All
     * resource cleanup is going to happen through process exit.
     */
    if (proc_exit_inprogress)
        return;
 
    llvm_enter_fatal_on_oom();
 
    if (llvm_context->module)
    {
        LLVMDisposeModule(llvm_context->module);
        llvm_context->module = NULL;
    }
 
    foreach(lc, llvm_context->handles)
    {
        LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
 
#if LLVM_VERSION_MAJOR > 11
        {
            LLVMOrcExecutionSessionRef ee;
            LLVMOrcSymbolStringPoolRef sp;
 
            LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
            LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
 
            /*
             * Without triggering cleanup of the string pool, we'd leak
             * memory. It'd be sufficient to do this far less often, but in
             * experiments the required time was small enough to just always
             * do it.
             */
            ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
            sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
            LLVMOrcSymbolStringPoolClearDeadEntries(sp);
        }
#else                           /* LLVM_VERSION_MAJOR > 11 */
        {
            LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
        }
#endif                          /* LLVM_VERSION_MAJOR > 11 */
 
        pfree(jit_handle);
    }
    list_free(llvm_context->handles);
    llvm_context->handles = NIL;
}

References lfirst, list_free(), llvm_enter_fatal_on_oom(), NIL, LLVMJitHandle::orc_handle, pfree(), proc_exit_inprogress, and LLVMJitHandle::stack.

Referenced by _PG_jit_provider_init().

llvm_resolve_symbol()

static uint64_t llvm_resolve_symbol ( const char *  name, void *  ctx  )

static

Definition at line 1079 of file llvmjit.c.

{
    uintptr_t   addr;
    char       *funcname;
    char       *modname;
 
    /*
     * macOS prefixes all object level symbols with an underscore. But neither
     * dlsym() nor PG's inliner expect that. So undo.
     */
#if defined(__darwin__)
    if (symname[0] != '_')
        elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
    symname++;
#endif
 
    llvm_split_symbol_name(symname, &modname, &funcname);
 
    /* functions that aren't resolved to names shouldn't ever get here */
    Assert(funcname);
 
    if (modname)
        addr = (uintptr_t) load_external_function(modname, funcname,
                                                  true, NULL);
    else
        addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
 
    pfree(funcname);
    if (modname)
        pfree(modname);
 
    /* let LLVM will error out - should never happen */
    if (!addr)
        elog(WARNING, "failed to resolve name %s", symname);
 
    return (uint64_t) addr;
}

References Assert(), elog(), ERROR, funcname, llvm_split_symbol_name(), load_external_function(), pfree(), and WARNING.

Referenced by llvm_compile_module().

llvm_session_initialize()

static void llvm_session_initialize ( void  )

static

Definition at line 783 of file llvmjit.c.

{
    MemoryContext oldcontext;
    char       *error = NULL;
    char       *cpu = NULL;
    char       *features = NULL;
    LLVMTargetMachineRef opt0_tm;
    LLVMTargetMachineRef opt3_tm;
 
    if (llvm_session_initialized)
        return;
 
    oldcontext = MemoryContextSwitchTo(TopMemoryContext);
 
    LLVMInitializeNativeTarget();
    LLVMInitializeNativeAsmPrinter();
    LLVMInitializeNativeAsmParser();
 
    /*
     * When targeting an LLVM version with opaque pointers enabled by
     * default, turn them off for the context we build our code in.  We don't
     * need to do so for other contexts (e.g. llvm_ts_context).  Once the IR is
     * generated, it carries the necessary information.
     */
#if LLVM_VERSION_MAJOR > 14
    LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
#endif
 
    /*
     * Synchronize types early, as that also includes inferring the target
     * triple.
     */
    llvm_create_types();
 
    if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
    {
        elog(FATAL, "failed to query triple %s", error);
    }
 
    /*
     * We want the generated code to use all available features. Therefore
     * grab the host CPU string and detect features of the current CPU. The
     * latter is needed because some CPU architectures default to enabling
     * features not all CPUs have (weird, huh).
     */
    cpu = LLVMGetHostCPUName();
    features = LLVMGetHostCPUFeatures();
    elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
         cpu, features);
 
    opt0_tm =
        LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
                                LLVMCodeGenLevelNone,
                                LLVMRelocDefault,
                                LLVMCodeModelJITDefault);
    opt3_tm =
        LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
                                LLVMCodeGenLevelAggressive,
                                LLVMRelocDefault,
                                LLVMCodeModelJITDefault);
 
    LLVMDisposeMessage(cpu);
    cpu = NULL;
    LLVMDisposeMessage(features);
    features = NULL;
 
    /* force symbols in main binary to be loaded */
    LLVMLoadLibraryPermanently(NULL);
 
#if LLVM_VERSION_MAJOR > 11
    {
        llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
 
        llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
        opt0_tm = 0;
 
        llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
        opt3_tm = 0;
    }
#else                           /* LLVM_VERSION_MAJOR > 11 */
    {
        llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
        llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
 
#if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
        if (jit_debugging_support)
        {
            LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
 
            LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
            LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
        }
#endif
#if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
        if (jit_profiling_support)
        {
            LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
 
            LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
            LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
        }
#endif
    }
#endif                          /* LLVM_VERSION_MAJOR > 11 */
 
    on_proc_exit(llvm_shutdown, 0);
 
    llvm_session_initialized = true;
 
    MemoryContextSwitchTo(oldcontext);
}

References DEBUG2, elog(), error(), FATAL, jit_debugging_support, jit_profiling_support, llvm_create_types(), llvm_opt0_orc, llvm_opt3_orc, llvm_session_initialized, llvm_shutdown(), llvm_targetref, llvm_triple, MemoryContextSwitchTo(), on_proc_exit(), and TopMemoryContext.

Referenced by llvm_create_context().

llvm_shutdown()

static void llvm_shutdown ( int  code, Datum  arg  )

static

Definition at line 896 of file llvmjit.c.

{
    /*
     * If llvm_shutdown() is reached while in a fatal-on-oom section an error
     * has occurred in the middle of LLVM code. It is not safe to call back
     * into LLVM (which is why a FATAL error was thrown).
     *
     * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
     * profiling data won't be written out.
     */
    if (llvm_in_fatal_on_oom())
    {
        Assert(proc_exit_inprogress);
        return;
    }
 
#if LLVM_VERSION_MAJOR > 11
    {
        if (llvm_opt3_orc)
        {
            LLVMOrcDisposeLLJIT(llvm_opt3_orc);
            llvm_opt3_orc = NULL;
        }
        if (llvm_opt0_orc)
        {
            LLVMOrcDisposeLLJIT(llvm_opt0_orc);
            llvm_opt0_orc = NULL;
        }
        if (llvm_ts_context)
        {
            LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
            llvm_ts_context = NULL;
        }
    }
#else                           /* LLVM_VERSION_MAJOR > 11 */
    {
        /* unregister profiling support, needs to be flushed to be useful */
 
        if (llvm_opt3_orc)
        {
#if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
            if (jit_profiling_support)
                LLVMOrcUnregisterPerf(llvm_opt3_orc);
#endif
            LLVMOrcDisposeInstance(llvm_opt3_orc);
            llvm_opt3_orc = NULL;
        }
 
        if (llvm_opt0_orc)
        {
#if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
            if (jit_profiling_support)
                LLVMOrcUnregisterPerf(llvm_opt0_orc);
#endif
            LLVMOrcDisposeInstance(llvm_opt0_orc);
            llvm_opt0_orc = NULL;
        }
    }
#endif                          /* LLVM_VERSION_MAJOR > 11 */
}

References Assert(), jit_profiling_support, llvm_in_fatal_on_oom(), llvm_opt0_orc, llvm_opt3_orc, and proc_exit_inprogress.

Referenced by llvm_session_initialize().

llvm_split_symbol_name()

void llvm_split_symbol_name	(	const char *	name,
		char **	modname,
		char **	funcname
	)

Definition at line 1045 of file llvmjit.c.

{
    *modname = NULL;
    *funcname = NULL;
 
    /*
     * Module function names are pgextern.$module.$funcname
     */
    if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
    {
        /*
         * Symbol names cannot contain a ., therefore we can split based on
         * first and last occurrence of one.
         */
        *funcname = rindex(name, '.');
        (*funcname)++;          /* jump over . */
 
        *modname = pnstrdup(name + strlen("pgextern."),
                            *funcname - name - strlen("pgextern.") - 1);
        Assert(funcname);
 
        *funcname = pstrdup(*funcname);
    }
    else
    {
        *modname = NULL;
        *funcname = pstrdup(name);
    }
}

References Assert(), funcname, name, pnstrdup(), and pstrdup().

Referenced by llvm_build_inline_plan(), llvm_execute_inline_plan(), and llvm_resolve_symbol().

load_return_type()

static LLVMTypeRef load_return_type ( LLVMModuleRef  mod, const char *  name  )

static

Definition at line 959 of file llvmjit.c.

{
    LLVMValueRef value;
    LLVMTypeRef typ;
 
    /* this'll return a *pointer* to the function */
    value = LLVMGetNamedFunction(mod, name);
    if (!value)
        elog(ERROR, "function %s is unknown", name);
 
    /* get type of function pointer */
    typ = LLVMTypeOf(value);
    Assert(typ != NULL);
    /* dereference pointer */
    typ = LLVMGetElementType(typ);
    Assert(typ != NULL);
    /* and look at return type */
    typ = LLVMGetReturnType(typ);
    Assert(typ != NULL);
 
    return typ;
}

References Assert(), elog(), ERROR, name, and value.

Referenced by llvm_create_types().

Variable Documentation

AttributeTemplate

Datum AttributeTemplate

Definition at line 89 of file llvmjit.c.

Referenced by llvm_create_types(), and slot_compile_deform().

llvm_generation

size_t llvm_generation = 0

static

Definition at line 94 of file llvmjit.c.

Referenced by llvm_mutable_module().

llvm_layout

const char* llvm_layout = NULL

static

Definition at line 96 of file llvmjit.c.

Referenced by llvm_create_types(), and llvm_mutable_module().

llvm_opt0_orc

LLVMOrcJITStackRef llvm_opt0_orc

static

Definition at line 105 of file llvmjit.c.

Referenced by llvm_compile_module(), llvm_get_function(), llvm_session_initialize(), and llvm_shutdown().

llvm_opt3_orc

LLVMOrcJITStackRef llvm_opt3_orc

static

Definition at line 106 of file llvmjit.c.

Referenced by llvm_compile_module(), llvm_get_function(), llvm_session_initialize(), and llvm_shutdown().

llvm_session_initialized

bool llvm_session_initialized = false

static

Definition at line 93 of file llvmjit.c.

Referenced by llvm_session_initialize().

llvm_targetref

LLVMTargetRef llvm_targetref

static

Definition at line 99 of file llvmjit.c.

Referenced by llvm_session_initialize().

llvm_triple

const char* llvm_triple = NULL

static

Definition at line 95 of file llvmjit.c.

Referenced by llvm_create_types(), llvm_mutable_module(), and llvm_session_initialize().

llvm_types_module

LLVMModuleRef llvm_types_module = NULL

Definition at line 91 of file llvmjit.c.

Referenced by llvm_create_types(), llvm_pg_func(), and llvm_pg_var_type().

PG_MODULE_MAGIC

PG_MODULE_MAGIC

Definition at line 124 of file llvmjit.c.

StructAggState

LLVMTypeRef StructAggState

Definition at line 85 of file llvmjit.c.

Referenced by llvm_create_types().

StructAggStatePerGroupData

LLVMTypeRef StructAggStatePerGroupData

Definition at line 86 of file llvmjit.c.

Referenced by llvm_create_types().

StructAggStatePerTransData

LLVMTypeRef StructAggStatePerTransData

Definition at line 87 of file llvmjit.c.

Referenced by llvm_create_types().

StructBlockId

LLVMTypeRef StructBlockId

Definition at line 71 of file llvmjit.c.

StructExprContext

LLVMTypeRef StructExprContext

Definition at line 82 of file llvmjit.c.

Referenced by llvm_create_types().

StructExprEvalStep

LLVMTypeRef StructExprEvalStep

Definition at line 83 of file llvmjit.c.

Referenced by build_EvalXFuncInt(), and llvm_create_types().

StructExprState

LLVMTypeRef StructExprState

Definition at line 84 of file llvmjit.c.

Referenced by llvm_create_types().

StructFmgrInfo

LLVMTypeRef StructFmgrInfo

Definition at line 80 of file llvmjit.c.

StructFormPgAttribute

LLVMTypeRef StructFormPgAttribute

Definition at line 72 of file llvmjit.c.

StructFunctionCallInfoData

LLVMTypeRef StructFunctionCallInfoData

Definition at line 81 of file llvmjit.c.

Referenced by BuildV1Call(), and llvm_create_types().

StructHeapTupleData

LLVMTypeRef StructHeapTupleData

Definition at line 68 of file llvmjit.c.

Referenced by llvm_create_types().

StructHeapTupleDataChoice

LLVMTypeRef StructHeapTupleDataChoice

Definition at line 67 of file llvmjit.c.

StructHeapTupleFields

LLVMTypeRef StructHeapTupleFields

Definition at line 65 of file llvmjit.c.

StructHeapTupleFieldsField3

LLVMTypeRef StructHeapTupleFieldsField3

Definition at line 64 of file llvmjit.c.

StructHeapTupleHeaderData

LLVMTypeRef StructHeapTupleHeaderData

Definition at line 66 of file llvmjit.c.

StructHeapTupleTableSlot

LLVMTypeRef StructHeapTupleTableSlot

Definition at line 76 of file llvmjit.c.

Referenced by llvm_create_types(), and slot_compile_deform().

StructItemPointerData

LLVMTypeRef StructItemPointerData

Definition at line 70 of file llvmjit.c.

StructMemoryContextData

LLVMTypeRef StructMemoryContextData

Definition at line 78 of file llvmjit.c.

Referenced by llvm_create_types().

StructMinimalTupleData

LLVMTypeRef StructMinimalTupleData

Definition at line 69 of file llvmjit.c.

StructMinimalTupleTableSlot

LLVMTypeRef StructMinimalTupleTableSlot

Definition at line 77 of file llvmjit.c.

Referenced by llvm_create_types(), and slot_compile_deform().

StructNullableDatum

LLVMTypeRef StructNullableDatum

Definition at line 63 of file llvmjit.c.

Referenced by llvm_create_types().

StructPGFinfoRecord

LLVMTypeRef StructPGFinfoRecord

Definition at line 79 of file llvmjit.c.

StructTupleConstr

LLVMTypeRef StructTupleConstr

Definition at line 73 of file llvmjit.c.

StructTupleDescData

LLVMTypeRef StructTupleDescData

Definition at line 74 of file llvmjit.c.

Referenced by llvm_create_types().

StructTupleTableSlot

LLVMTypeRef StructTupleTableSlot

Definition at line 75 of file llvmjit.c.

Referenced by llvm_create_types(), and slot_compile_deform().

TypeParamBool

LLVMTypeRef TypeParamBool

Definition at line 60 of file llvmjit.c.

Referenced by llvm_create_types().

TypePGFunction

LLVMTypeRef TypePGFunction

Definition at line 62 of file llvmjit.c.

Referenced by llvm_create_types(), and llvm_function_reference().

TypeSizeT

LLVMTypeRef TypeSizeT

Definition at line 59 of file llvmjit.c.

Referenced by llvm_create_types(), and slot_compile_deform().

TypeStorageBool

LLVMTypeRef TypeStorageBool

Definition at line 61 of file llvmjit.c.

Referenced by llvm_create_types().