llvmjit_inline.cpp File Reference

#include "postgres.h"
#include "jit/llvmjit.h"
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "common/string.h"
#include "miscadmin.h"
#include "storage/fd.h"
#include <llvm-c/Core.h>
#include <llvm-c/BitReader.h>
#include <llvm/ADT/SetVector.h>
#include <llvm/ADT/StringSet.h>
#include <llvm/ADT/StringMap.h>
#include <llvm/Analysis/ModuleSummaryAnalysis.h>
#include <llvm/Bitcode/ReaderWriter.h>
#include <llvm/Support/Error.h>
#include <llvm/IR/Attributes.h>
#include <llvm/IR/DebugInfo.h>
#include <llvm/IR/IntrinsicInst.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/ModuleSummaryIndex.h>
#include <llvm/Linker/IRMover.h>
#include <llvm/Support/ManagedStatic.h>
#include <llvm/Support/MemoryBuffer.h>

Include dependency graph for llvmjit_inline.cpp:

Go to the source code of this file.

Data Structures
struct	InlineWorkListItem
struct	FunctionInlineState

Macros
#define	ilog(...)   (void) 0
#define	IRMOVE_PARAMS
#define	hasFnAttr   hasFnAttribute

Typedefs
typedef llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 >	InlineSearchPath
typedef struct InlineWorkListItem	InlineWorkListItem
typedef llvm::SmallVector< InlineWorkListItem, 128 >	InlineWorkList
typedef struct FunctionInlineState	FunctionInlineState
typedef llvm::StringMap< FunctionInlineState >	FunctionInlineStates
typedef llvm::StringMap< llvm::StringSet<> >	ImportMapTy
typedef llvm::StringMap< std::unique_ptr< llvm::Module > >	ModuleCache
typedef llvm::StringMap< std::unique_ptr< llvm::ModuleSummaryIndex > >	SummaryCache

Functions
static std::unique_ptr< ImportMapTy >	llvm_build_inline_plan (LLVMContextRef lc, llvm::Module *mod)
static void	llvm_execute_inline_plan (llvm::Module *mod, ImportMapTy *globalsToInline)
static llvm::Module *	load_module_cached (LLVMContextRef c, llvm::StringRef modPath)
static std::unique_ptr< llvm::Module >	load_module (LLVMContextRef c, llvm::StringRef Identifier)
static std::unique_ptr< llvm::ModuleSummaryIndex >	llvm_load_summary (llvm::StringRef path)
static llvm::Function *	create_redirection_function (std::unique_ptr< llvm::Module > &importMod, llvm::Function *F, llvm::StringRef Name)
static bool	function_inlinable (llvm::Function &F, int threshold, FunctionInlineStates &functionState, InlineWorkList &worklist, InlineSearchPath &searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &visitedFunctions, int &running_instcount, llvm::StringSet<> &importVars)
static void	function_references (llvm::Function &F, int &running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &referencedFunctions)
static void	add_module_to_inline_search_path (InlineSearchPath &path, llvm::StringRef modpath)
static llvm::SmallVector< llvm::GlobalValueSummary *, 1 >	summaries_for_guid (const InlineSearchPath &path, llvm::GlobalValue::GUID guid)
void	llvm_inline_reset_caches (void)
void	llvm_inline (LLVMModuleRef M)

Variables
const float	inline_cost_decay_factor = 0.5
const int	inline_initial_cost = 150
llvm::ManagedStatic< ModuleCache >	module_cache
llvm::ManagedStatic< SummaryCache >	summary_cache

Macro Definition Documentation

hasFnAttr

#define hasFnAttr   hasFnAttribute

ilog

#define ilog

(

...

)

(void) 0

Definition at line 152 of file llvmjit_inline.cpp.

IRMOVE_PARAMS

#define IRMOVE_PARAMS

Typedef Documentation

FunctionInlineState

typedef struct FunctionInlineState FunctionInlineState

FunctionInlineStates

typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates

Definition at line 95 of file llvmjit_inline.cpp.

ImportMapTy

typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy

Definition at line 101 of file llvmjit_inline.cpp.

InlineSearchPath

typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath

Definition at line 72 of file llvmjit_inline.cpp.

InlineWorkList

typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList

Definition at line 82 of file llvmjit_inline.cpp.

InlineWorkListItem

typedef struct InlineWorkListItem InlineWorkListItem

ModuleCache

typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache

Definition at line 111 of file llvmjit_inline.cpp.

SummaryCache

typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache

Definition at line 113 of file llvmjit_inline.cpp.

Function Documentation

add_module_to_inline_search_path()

static void add_module_to_inline_search_path ( InlineSearchPath &  path, llvm::StringRef  modpath  )

static

Definition at line 821 of file llvmjit_inline.cpp.

{
    /* only extension in libdir are candidates for inlining for now */
    if (!modpath.startswith("$libdir/"))
        return;
 
    /* if there's no match, attempt to load */
    auto it = summary_cache->find(modpath);
    if (it == summary_cache->end())
    {
        std::string path(modpath);
        path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
        path += ".index.bc";
        (*summary_cache)[modpath] = llvm_load_summary(path);
        it = summary_cache->find(modpath);
    }
 
    Assert(it != summary_cache->end());
 
    /* if the entry isn't NULL, it's validly loaded */
    if (it->second)
        searchpath.push_back(it->second.get());
}

References Assert(), llvm_load_summary(), pkglib_path, and summary_cache.

Referenced by llvm_build_inline_plan().

create_redirection_function()

static llvm::Function * create_redirection_function ( std::unique_ptr< llvm::Module > &  importMod, llvm::Function *  F, llvm::StringRef  Name  )

static

Definition at line 881 of file llvmjit_inline.cpp.

{
    typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
 
    llvm::LLVMContext &Context = F->getContext();
    llvm::IRBuilder<> Builder(Context);
    llvm::Function *AF;
    llvm::BasicBlock *BB;
    llvm::CallInst *fwdcall;
#if LLVM_VERSION_MAJOR < 14
    llvm::Attribute inlineAttribute;
#endif
 
    AF = llvm::Function::Create(F->getFunctionType(),
                                LinkageTypes::AvailableExternallyLinkage,
                                Name, importMod.get());
    BB = llvm::BasicBlock::Create(Context, "entry", AF);
 
    Builder.SetInsertPoint(BB);
    fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
#if LLVM_VERSION_MAJOR < 14
    inlineAttribute = llvm::Attribute::get(Context,
                                           llvm::Attribute::AlwaysInline);
    fwdcall->addAttribute(~0U, inlineAttribute);
#else
    fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
#endif
    Builder.CreateRet(fwdcall);
 
    return AF;
}

References F.

Referenced by llvm_execute_inline_plan().

function_inlinable()

static bool function_inlinable ( llvm::Function &  F, int  threshold, FunctionInlineStates &  functionState, InlineWorkList &  worklist, InlineSearchPath &  searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &  visitedFunctions, int &  running_instcount, llvm::StringSet<> &  importVars  )

static

Definition at line 582 of file llvmjit_inline.cpp.

{
    int subThreshold = threshold * inline_cost_decay_factor;
    llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
    llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
 
    /* can't rely on what may be inlined */
    if (F.isInterposable())
        return false;
 
    /*
     * Can't rely on function being present. Alternatively we could create a
     * static version of these functions?
     */
    if (F.hasAvailableExternallyLinkage())
        return false;
 
    ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
 
    if (F.materialize())
        elog(FATAL, "failed to materialize metadata");
 
#if LLVM_VERSION_MAJOR < 14
#define hasFnAttr hasFnAttribute
#endif
 
    if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
    {
        ilog(DEBUG1, "ineligibile to import %s due to noinline",
             F.getName().data());
        return false;
    }
 
    function_references(F, running_instcount, referencedVars, referencedFunctions);
 
    for (llvm::GlobalVariable* rv: referencedVars)
    {
        if (rv->materialize())
            elog(FATAL, "failed to materialize metadata");
 
        /*
         * Don't inline functions that access thread local variables.  That
         * doesn't work on current LLVM releases (but might in future).
         */
        if (rv->isThreadLocal())
        {
            ilog(DEBUG1, "cannot inline %s due to thread-local variable %s",
                 F.getName().data(), rv->getName().data());
            return false;
        }
 
        /*
         * Never want to inline externally visible vars, cheap enough to
         * reference.
         */
        if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
            continue;
 
        /*
         * If variable is file-local, we need to inline it, to be able to
         * inline the function itself. Can't do that if the variable can be
         * modified, because they'd obviously get out of sync.
         *
         * XXX: Currently not a problem, but there'd be problems with
         * nontrivial initializers if they were allowed for postgres.
         */
        if (!rv->isConstant())
        {
            ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
                 F.getName().data(), rv->getName().data());
            return false;
        }
 
        ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
             rv->getName().data(), (int)rv->getLinkage());
 
        importVars.insert(rv->getName());
        /* small cost attributed to each cloned global */
        running_instcount += 5;
    }
 
    visitedFunctions.insert(&F);
 
    /*
     * Check referenced functions. Check whether used static ones are
     * inlinable, and remember external ones for inlining.
     */
    for (llvm::Function* referencedFunction: referencedFunctions)
    {
        llvm::StringSet<> recImportVars;
 
        if (referencedFunction->materialize())
            elog(FATAL, "failed to materialize metadata");
 
        if (referencedFunction->isIntrinsic())
            continue;
 
        /* if already visited skip, otherwise remember */
        if (!visitedFunctions.insert(referencedFunction).second)
            continue;
 
        /*
         * We don't inline external functions directly here, instead we put
         * them on the worklist if appropriate and check them from
         * llvm_build_inline_plan().
         */
        if (referencedFunction->hasExternalLinkage())
        {
            llvm::StringRef funcName = referencedFunction->getName();
 
            /*
             * Don't bother checking for inlining if remaining cost budget is
             * very small.
             */
            if (subThreshold < 5)
                continue;
 
            auto it = functionStates.find(funcName);
            if (it == functionStates.end())
            {
                FunctionInlineState inlineState;
 
                inlineState.costLimit = subThreshold;
                inlineState.processed = false;
                inlineState.inlined = false;
                inlineState.allowReconsidering = false;
 
                functionStates[funcName] = inlineState;
                worklist.push_back({funcName, searchpath});
 
                ilog(DEBUG1,
                     "considering extern function %s at %d for inlining",
                     funcName.data(), subThreshold);
            }
            else if (!it->second.inlined &&
                     (!it->second.processed || it->second.allowReconsidering) &&
                     it->second.costLimit < subThreshold)
            {
                /*
                 * Update inlining threshold if higher. Need to re-queue
                 * to be processed if already processed with lower
                 * threshold.
                 */
                if (it->second.processed)
                {
                    ilog(DEBUG1,
                         "reconsidering extern function %s at %d for inlining, increasing from %d",
                         funcName.data(), subThreshold, it->second.costLimit);
 
                    it->second.processed = false;
                    it->second.allowReconsidering = false;
                    worklist.push_back({funcName, searchpath});
                }
                it->second.costLimit = subThreshold;
            }
            continue;
        }
 
        /* can't rely on what may be inlined */
        if (referencedFunction->isInterposable())
            return false;
 
        if (!function_inlinable(*referencedFunction,
                                subThreshold,
                                functionStates,
                                worklist,
                                searchpath,
                                visitedFunctions,
                                running_instcount,
                                recImportVars))
        {
            ilog(DEBUG1,
                 "cannot inline %s due to required function %s not being inlinable",
                 F.getName().data(), referencedFunction->getName().data());
            return false;
        }
 
        /* import referenced function itself */
        importVars.insert(referencedFunction->getName());
 
        /* import referenced function and its dependents */
        for (auto& recImportVar : recImportVars)
            importVars.insert(recImportVar.first());
    }
 
    return true;
}

References FunctionInlineState::allowReconsidering, FunctionInlineState::costLimit, DEBUG1, elog(), F, FATAL, function_references(), ilog, inline_cost_decay_factor, FunctionInlineState::inlined, and FunctionInlineState::processed.

Referenced by llvm_build_inline_plan().

function_references()

static void function_references ( llvm::Function &  F, int &  running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &  referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &  referencedFunctions  )

static

Definition at line 525 of file llvmjit_inline.cpp.

{
    llvm::SmallPtrSet<const llvm::User *, 32> Visited;
 
    for (llvm::BasicBlock &BB : F)
    {
        for (llvm::Instruction &I : BB)
        {
            if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
                continue;
 
            llvm::SmallVector<llvm::User *, 8> Worklist;
            Worklist.push_back(&I);
 
            running_instcount++;
 
            while (!Worklist.empty()) {
                llvm::User *U = Worklist.pop_back_val();
 
                /* visited before */
                if (!Visited.insert(U).second)
                    continue;
 
                for (auto &OI : U->operands()) {
                    llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
                    if (!Operand)
                        continue;
                    if (llvm::isa<llvm::BlockAddress>(Operand))
                        continue;
                    if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
                        referencedVars.insert(GV);
                        if (GV->hasInitializer())
                            Worklist.push_back(GV->getInitializer());
                        continue;
                    }
                    if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
                        referencedFunctions.insert(CF);
                        continue;
                    }
                    Worklist.push_back(Operand);
                }
            }
        }
    }
}

References F, and I.

Referenced by function_inlinable().

llvm_build_inline_plan()

static std::unique_ptr< ImportMapTy > llvm_build_inline_plan ( LLVMContextRef  lc, llvm::Module *  mod  )

static

Definition at line 188 of file llvmjit_inline.cpp.

{
    std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
    FunctionInlineStates functionStates;
    InlineWorkList worklist;
 
    InlineSearchPath defaultSearchPath;
 
    /* attempt to add module to search path */
    add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
    /* if postgres isn't available, no point continuing */
    if (defaultSearchPath.empty())
        return nullptr;
 
    /*
     * Start inlining with current references to external functions by putting
     * them on the inlining worklist. If, during inlining of those, new extern
     * functions need to be inlined, they'll also be put there, with a lower
     * priority.
     */
    for (const llvm::Function &funcDecl : mod->functions())
    {
        InlineWorkListItem item = {};
        FunctionInlineState inlineState = {};
 
        /* already has a definition */
        if (!funcDecl.isDeclaration())
            continue;
 
        /* llvm provides implementation */
        if (funcDecl.isIntrinsic())
            continue;
 
        item.symbolName = funcDecl.getName();
        item.searchpath = defaultSearchPath;
        worklist.push_back(item);
        inlineState.costLimit = inline_initial_cost;
        inlineState.processed = false;
        inlineState.inlined = false;
        inlineState.allowReconsidering = false;
        functionStates[funcDecl.getName()] = inlineState;
    }
 
    /*
     * Iterate over pending worklist items, look them up in index, check
     * whether they should be inlined.
     */
    while (!worklist.empty())
    {
        InlineWorkListItem item = worklist.pop_back_val();
        llvm::StringRef symbolName = item.symbolName;
        char *cmodname;
        char *cfuncname;
        FunctionInlineState &inlineState = functionStates[symbolName];
        llvm::GlobalValue::GUID funcGUID;
 
        llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
 
        funcGUID = llvm::GlobalValue::getGUID(cfuncname);
 
        /* already processed */
        if (inlineState.processed)
            continue;
 
 
        if (cmodname)
            add_module_to_inline_search_path(item.searchpath, cmodname);
 
        /*
         * Iterate over all known definitions of function, via the index. Then
         * look up module(s), check if function actually is defined (there
         * could be hash conflicts).
         */
        for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
        {
            const llvm::FunctionSummary *fs;
            llvm::StringRef modPath = gvs->modulePath();
            llvm::Module *defMod;
            llvm::Function *funcDef;
 
            fs = llvm::cast<llvm::FunctionSummary>(gvs);
 
#if LLVM_VERSION_MAJOR > 3
            if (gvs->notEligibleToImport())
            {
                ilog(DEBUG1, "ineligibile to import %s due to summary",
                     symbolName.data());
                continue;
            }
#endif
 
            if ((int) fs->instCount() > inlineState.costLimit)
            {
                ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
                     symbolName.data(), fs->instCount(), inlineState.costLimit);
                inlineState.allowReconsidering = true;
                continue;
            }
 
            defMod = load_module_cached(lc, modPath);
            if (defMod->materializeMetadata())
                elog(FATAL, "failed to materialize metadata");
 
            funcDef = defMod->getFunction(cfuncname);
 
            /*
             * This can happen e.g. in case of a hash collision of the
             * function's name.
             */
            if (!funcDef)
                continue;
 
            if (funcDef->materialize())
                elog(FATAL, "failed to materialize metadata");
 
            Assert(!funcDef->isDeclaration());
            Assert(funcDef->hasExternalLinkage());
 
            llvm::StringSet<> importVars;
            llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
            int running_instcount = 0;
 
            /*
             * Check whether function, and objects it depends on, are
             * inlinable.
             */
            if (function_inlinable(*funcDef,
                                   inlineState.costLimit,
                                   functionStates,
                                   worklist,
                                   item.searchpath,
                                   visitedFunctions,
                                   running_instcount,
                                   importVars))
            {
                /*
                 * Check whether function and all its dependencies are too
                 * big. Dependencies already counted for other functions that
                 * will get inlined are not counted again. While this make
                 * things somewhat order dependent, I can't quite see a point
                 * in a different behaviour.
                 */
                if (running_instcount > inlineState.costLimit)
                {
                    ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
                         symbolName.data(), running_instcount, inlineState.costLimit);
                    inlineState.allowReconsidering = true;
                    continue;
                }
 
                ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
                     symbolName.data(), running_instcount, fs->instCount());
 
                /* import referenced function itself */
                importVars.insert(symbolName);
 
                {
                    llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
                    for (auto& importVar : importVars)
                        modGlobalsToInline.insert(importVar.first());
                    Assert(modGlobalsToInline.size() > 0);
                }
 
                /* mark function as inlined */
                inlineState.inlined = true;
 
                /*
                 * Found definition to inline, don't look for further
                 * potential definitions.
                 */
                break;
            }
            else
            {
                ilog(DEBUG1, "had to skip inlining %s",
                     symbolName.data());
 
                /* It's possible there's another definition that's inlinable. */
            }
        }
 
        /*
         * Signal that we're done with symbol, whether successful (inlined =
         * true above) or not.
         */
        inlineState.processed = true;
    }
 
    return globalsToInline;
}

References add_module_to_inline_search_path(), FunctionInlineState::allowReconsidering, Assert(), FunctionInlineState::costLimit, DEBUG1, elog(), FATAL, function_inlinable(), ilog, inline_initial_cost, FunctionInlineState::inlined, llvm_split_symbol_name(), load_module_cached(), FunctionInlineState::processed, InlineWorkListItem::searchpath, summaries_for_guid(), and InlineWorkListItem::symbolName.

Referenced by llvm_inline().

llvm_execute_inline_plan()

static void llvm_execute_inline_plan ( llvm::Module *  mod, ImportMapTy *  globalsToInline  )

static

Definition at line 384 of file llvmjit_inline.cpp.

{
    llvm::IRMover Mover(*mod);
 
    for (const auto& toInline : *globalsToInline)
    {
        const llvm::StringRef& modPath = toInline.first();
        const llvm::StringSet<>& modGlobalsToInline = toInline.second;
        llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
 
        Assert(module_cache->count(modPath));
        std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
        module_cache->erase(modPath);
 
        if (modGlobalsToInline.empty())
            continue;
 
        for (auto &glob: modGlobalsToInline)
        {
            llvm::StringRef SymbolName = glob.first();
            char *modname;
            char *funcname;
 
            llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
 
            llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
 
            if (!valueToImport)
                elog(FATAL, "didn't refind value %s to import", SymbolName.data());
 
            /*
             * For functions (global vars are only inlined if already static),
             * mark imported variables as being clones from other
             * functions. That a) avoids symbol conflicts b) allows the
             * optimizer to perform inlining.
            */
            if (llvm::isa<llvm::Function>(valueToImport))
            {
                llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
                typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
 
                /*
                 * Per-function info isn't necessarily stripped yet, as the
                 * module is lazy-loaded when stripped above.
                 */
                llvm::stripDebugInfo(*F);
 
                /*
                 * If the to-be-imported function is one referenced including
                 * its module name, create a tiny inline function that just
                 * forwards the call. One might think a GlobalAlias would do
                 * the trick, but a) IRMover doesn't override a declaration
                 * with an alias pointing to a definition (instead renaming
                 * it), b) Aliases can't be AvailableExternally.
                 */
                if (modname)
                {
                    llvm::Function *AF;
 
                    AF = create_redirection_function(importMod, F, SymbolName);
 
                    GlobalsToImport.insert(AF);
                    llvm::stripDebugInfo(*AF);
                }
 
                if (valueToImport->hasExternalLinkage())
                {
                    valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
                }
            }
 
            GlobalsToImport.insert(valueToImport);
            ilog(DEBUG1, "performing import of %s %s",
                 modPath.data(), SymbolName.data());
 
        }
 
#if LLVM_VERSION_MAJOR > 4
#define IRMOVE_PARAMS , /*IsPerformingImport=*/false
#elif LLVM_VERSION_MAJOR > 3
#define IRMOVE_PARAMS , /*LinkModuleInlineAsm=*/false, /*IsPerformingImport=*/false
#else
#define IRMOVE_PARAMS
#endif
        if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
                       [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}
                       IRMOVE_PARAMS))
            elog(FATAL, "function import failed with linker error");
    }
}

References Assert(), create_redirection_function(), DEBUG1, elog(), F, FATAL, funcname, ilog, IRMOVE_PARAMS, llvm_split_symbol_name(), and module_cache.

Referenced by llvm_inline().

llvm_inline()

void llvm_inline

(

LLVMModuleRef

M

)

Definition at line 172 of file llvmjit_inline.cpp.

{
    LLVMContextRef lc = LLVMGetModuleContext(M);
    llvm::Module *mod = llvm::unwrap(M);
 
    std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(lc, mod);
    if (!globalsToInline)
        return;
    llvm_execute_inline_plan(mod, globalsToInline.get());
}

References llvm_build_inline_plan(), and llvm_execute_inline_plan().

Referenced by llvm_compile_module().

llvm_inline_reset_caches()

void llvm_inline_reset_caches

(

void

)

Definition at line 161 of file llvmjit_inline.cpp.

{
    module_cache->clear();
    summary_cache->clear();
}

References module_cache, and summary_cache.

Referenced by llvm_recreate_llvm_context().

llvm_load_summary()

static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary ( llvm::StringRef  path )

static

Definition at line 782 of file llvmjit_inline.cpp.

{
    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
        llvm::MemoryBuffer::getFile(path);
 
    if (std::error_code EC = MBOrErr.getError())
    {
        ilog(DEBUG1, "failed to open %s: %s", path.data(),
             EC.message().c_str());
    }
    else
    {
        llvm::MemoryBufferRef ref(*MBOrErr.get().get());
 
#if LLVM_VERSION_MAJOR > 3
        llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
            llvm::getModuleSummaryIndex(ref);
        if (IndexOrErr)
            return std::move(IndexOrErr.get());
        elog(FATAL, "failed to load summary \"%s\": %s",
             path.data(),
             toString(IndexOrErr.takeError()).c_str());
#else
        llvm::ErrorOr<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
            llvm::getModuleSummaryIndex(ref, [](const llvm::DiagnosticInfo &) {});
        if (IndexOrErr)
            return std::move(IndexOrErr.get());
        elog(FATAL, "failed to load summary \"%s\": %s",
             path.data(),
             IndexOrErr.getError().message().c_str());
#endif
    }
    return nullptr;
}

References DEBUG1, elog(), FATAL, and ilog.

Referenced by add_module_to_inline_search_path().

load_module()

static std::unique_ptr< llvm::Module > load_module ( LLVMContextRef  c, llvm::StringRef  Identifier  )

static

Definition at line 495 of file llvmjit_inline.cpp.

{
    LLVMMemoryBufferRef buf;
    LLVMModuleRef mod;
    char path[MAXPGPATH];
    char *msg;
 
    snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
 
    if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
        elog(FATAL, "failed to open bitcode file \"%s\": %s",
             path, msg);
    if (LLVMGetBitcodeModuleInContext2(lc, buf, &mod))
        elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
 
    /*
     * Currently there's no use in more detailed debug info for JITed
     * code. Until that changes, not much point in wasting memory and cycles
     * on processing debuginfo.
     */
    llvm::StripDebugInfo(*llvm::unwrap(mod));
 
    return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
}

References buf, elog(), FATAL, MAXPGPATH, pkglib_path, and snprintf.

Referenced by load_module_cached().

load_module_cached()

static llvm::Module * load_module_cached ( LLVMContextRef  c, llvm::StringRef  modPath  )

static

Definition at line 482 of file llvmjit_inline.cpp.

{
    auto it = module_cache->find(modPath);
    if (it == module_cache->end())
    {
        it = module_cache->insert(
            std::make_pair(modPath, load_module(lc, modPath))).first;
    }
 
    return it->second.get();
}

References load_module(), and module_cache.

Referenced by llvm_build_inline_plan().

summaries_for_guid()

static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid ( const InlineSearchPath &  path, llvm::GlobalValue::GUID  guid  )

static

Definition at line 850 of file llvmjit_inline.cpp.

{
    llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
 
    for (auto index : path)
    {
#if LLVM_VERSION_MAJOR > 4
        llvm::ValueInfo funcVI = index->getValueInfo(guid);
 
        /* if index doesn't know function, we don't have a body, continue */
        if (funcVI)
            for (auto &gv : funcVI.getSummaryList())
                matches.push_back(gv.get());
#else
        const llvm::const_gvsummary_iterator &I =
            index->findGlobalValueSummaryList(guid);
        if (I != index->end())
        {
            for (auto &gv : I->second)
                matches.push_back(gv.get());
        }
#endif
    }
 
    return matches;
}

References I.

Referenced by llvm_build_inline_plan().

Variable Documentation

inline_cost_decay_factor

const float inline_cost_decay_factor = 0.5

Definition at line 104 of file llvmjit_inline.cpp.

Referenced by function_inlinable().

inline_initial_cost

const int inline_initial_cost = 150

Definition at line 105 of file llvmjit_inline.cpp.

Referenced by llvm_build_inline_plan().

module_cache

llvm::ManagedStatic<ModuleCache> module_cache

Definition at line 112 of file llvmjit_inline.cpp.

Referenced by llvm_execute_inline_plan(), llvm_inline_reset_caches(), and load_module_cached().

summary_cache

llvm::ManagedStatic<SummaryCache> summary_cache

Definition at line 114 of file llvmjit_inline.cpp.

Referenced by add_module_to_inline_search_path(), and llvm_inline_reset_caches().