34 #include <sys/types.h>
42 #include <llvm-c/Core.h>
43 #include <llvm-c/BitReader.h>
48 #include <llvm/ADT/SetVector.h>
49 #include <llvm/ADT/StringSet.h>
50 #include <llvm/ADT/StringMap.h>
51 #include <llvm/Analysis/ModuleSummaryAnalysis.h>
52 #if LLVM_VERSION_MAJOR > 3
53 #include <llvm/Bitcode/BitcodeReader.h>
55 #include <llvm/Bitcode/ReaderWriter.h>
56 #include <llvm/Support/Error.h>
58 #include <llvm/IR/Attributes.h>
59 #include <llvm/IR/DebugInfo.h>
60 #include <llvm/IR/IntrinsicInst.h>
61 #include <llvm/IR/IRBuilder.h>
62 #include <llvm/IR/ModuleSummaryIndex.h>
63 #include <llvm/Linker/IRMover.h>
64 #include <llvm/Support/ManagedStatic.h>
65 #include <llvm/Support/MemoryBuffer.h>
80 llvm::SmallVector<llvm::ModuleSummaryIndex *, 2>
searchpath;
111 typedef llvm::StringMap<std::unique_ptr<llvm::Module> >
ModuleCache;
113 typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> >
SummaryCache;
122 static std::unique_ptr<llvm::Module>
load_module(llvm::StringRef Identifier);
123 static std::unique_ptr<llvm::ModuleSummaryIndex>
llvm_load_summary(llvm::StringRef path);
128 llvm::StringRef
Name);
135 llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
136 int &running_instcount,
137 llvm::StringSet<> &importVars);
139 int &running_instcount,
140 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
141 llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions);
144 static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
152 #define ilog(...) (void) 0
162 llvm::Module *mod = llvm::unwrap(M);
165 if (!globalsToInline)
174 static std::unique_ptr<ImportMapTy>
177 std::unique_ptr<ImportMapTy> globalsToInline(
new ImportMapTy());
186 if (defaultSearchPath.empty())
195 for (
const llvm::Function &funcDecl : mod->functions())
201 if (!funcDecl.isDeclaration())
205 if (funcDecl.isIntrinsic())
210 worklist.push_back(item);
215 functionStates[funcDecl.getName()] = inlineState;
222 while (!worklist.empty())
229 llvm::GlobalValue::GUID funcGUID;
233 funcGUID = llvm::GlobalValue::getGUID(cfuncname);
250 const llvm::FunctionSummary *fs;
251 llvm::StringRef modPath = gvs->modulePath();
252 llvm::Module *defMod;
253 llvm::Function *funcDef;
255 fs = llvm::cast<llvm::FunctionSummary>(gvs);
257 #if LLVM_VERSION_MAJOR > 3
258 if (gvs->notEligibleToImport())
260 ilog(
DEBUG1,
"ineligibile to import %s due to summary",
266 if ((
int) fs->instCount() > inlineState.
costLimit)
268 ilog(
DEBUG1,
"ineligibile to import %s due to early threshold: %u vs %u",
269 symbolName.data(), fs->instCount(), inlineState.
costLimit);
275 if (defMod->materializeMetadata())
276 elog(
FATAL,
"failed to materialize metadata");
278 funcDef = defMod->getFunction(cfuncname);
287 if (funcDef->materialize())
288 elog(
FATAL,
"failed to materialize metadata");
290 Assert(!funcDef->isDeclaration());
291 Assert(funcDef->hasExternalLinkage());
293 llvm::StringSet<> importVars;
294 llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
295 int running_instcount = 0;
317 if (running_instcount > inlineState.
costLimit)
319 ilog(
DEBUG1,
"skipping inlining of %s due to late threshold %d vs %d",
320 symbolName.data(), running_instcount, inlineState.
costLimit);
325 ilog(
DEBUG1,
"inline top function %s total_instcount: %d, partial: %d",
326 symbolName.data(), running_instcount, fs->instCount());
329 importVars.insert(symbolName);
332 llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
333 for (
auto& importVar : importVars)
334 modGlobalsToInline.insert(importVar.first());
335 Assert(modGlobalsToInline.size() > 0);
363 return globalsToInline;
373 llvm::IRMover Mover(*mod);
375 for (
const auto& toInline : *globalsToInline)
377 const llvm::StringRef& modPath = toInline.first();
378 const llvm::StringSet<>& modGlobalsToInline = toInline.second;
379 llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
382 std::unique_ptr<llvm::Module> importMod(std::move((*
module_cache)[modPath]));
385 if (modGlobalsToInline.empty())
388 for (
auto &glob: modGlobalsToInline)
390 llvm::StringRef SymbolName = glob.first();
396 llvm::GlobalValue *valueToImport = importMod->getNamedValue(
funcname);
399 elog(
FATAL,
"didn't refind value %s to import", SymbolName.data());
407 if (llvm::isa<llvm::Function>(valueToImport))
409 llvm::Function *
F = llvm::dyn_cast<llvm::Function>(valueToImport);
410 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
416 llvm::stripDebugInfo(*
F);
432 GlobalsToImport.insert(AF);
433 llvm::stripDebugInfo(*AF);
436 if (valueToImport->hasExternalLinkage())
438 valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
442 GlobalsToImport.insert(valueToImport);
444 modPath.data(), SymbolName.data());
448 #if LLVM_VERSION_MAJOR > 4
449 #define IRMOVE_PARAMS , false
450 #elif LLVM_VERSION_MAJOR > 3
451 #define IRMOVE_PARAMS , false, false
453 #define IRMOVE_PARAMS
455 if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
456 [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}
458 elog(
FATAL,
"function import failed with linker error");
475 std::make_pair(modPath,
load_module(modPath))).first;
478 return it->second.get();
481 static std::unique_ptr<llvm::Module>
484 LLVMMemoryBufferRef
buf;
491 if (LLVMCreateMemoryBufferWithContentsOfFile(path, &
buf, &msg))
492 elog(
FATAL,
"failed to open bitcode file \"%s\": %s",
494 if (LLVMGetBitcodeModuleInContext2(LLVMGetGlobalContext(),
buf, &mod))
495 elog(
FATAL,
"failed to parse bitcode in file \"%s\"", path);
502 llvm::StripDebugInfo(*llvm::unwrap(mod));
504 return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
513 int &running_instcount,
514 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
515 llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions)
517 llvm::SmallPtrSet<const llvm::User *, 32> Visited;
519 for (llvm::BasicBlock &BB :
F)
521 for (llvm::Instruction &
I : BB)
523 if (llvm::isa<llvm::DbgInfoIntrinsic>(
I))
526 llvm::SmallVector<llvm::User *, 8> Worklist;
527 Worklist.push_back(&
I);
531 while (!Worklist.empty()) {
532 llvm::User *U = Worklist.pop_back_val();
535 if (!Visited.insert(U).second)
538 for (
auto &OI : U->operands()) {
539 llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
542 if (llvm::isa<llvm::BlockAddress>(Operand))
544 if (
auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
545 referencedVars.insert(GV);
546 if (GV->hasInitializer())
547 Worklist.push_back(GV->getInitializer());
550 if (
auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
551 referencedFunctions.insert(CF);
554 Worklist.push_back(Operand);
574 llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
575 int &running_instcount,
576 llvm::StringSet<> &importVars)
579 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
580 llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
583 if (
F.isInterposable())
590 if (
F.hasAvailableExternallyLinkage())
593 ilog(
DEBUG1,
"checking inlinability of %s",
F.getName().data());
596 elog(
FATAL,
"failed to materialize metadata");
598 #if LLVM_VERSION_MAJOR < 14
599 #define hasFnAttr hasFnAttribute
602 if (
F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
604 ilog(
DEBUG1,
"ineligibile to import %s due to noinline",
611 for (llvm::GlobalVariable* rv: referencedVars)
613 if (rv->materialize())
614 elog(
FATAL,
"failed to materialize metadata");
620 if (rv->isThreadLocal())
622 ilog(
DEBUG1,
"cannot inline %s due to thread-local variable %s",
623 F.getName().data(), rv->getName().data());
631 if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
642 if (!rv->isConstant())
644 ilog(
DEBUG1,
"cannot inline %s due to uncloneable variable %s",
645 F.getName().data(), rv->getName().data());
649 ilog(
DEBUG1,
"memorizing global var %s linkage %d for inlining",
650 rv->getName().data(), (
int)rv->getLinkage());
652 importVars.insert(rv->getName());
654 running_instcount += 5;
657 visitedFunctions.insert(&
F);
663 for (llvm::Function* referencedFunction: referencedFunctions)
665 llvm::StringSet<> recImportVars;
667 if (referencedFunction->materialize())
668 elog(
FATAL,
"failed to materialize metadata");
670 if (referencedFunction->isIntrinsic())
674 if (!visitedFunctions.insert(referencedFunction).second)
682 if (referencedFunction->hasExternalLinkage())
684 llvm::StringRef funcName = referencedFunction->getName();
690 if (subThreshold < 5)
693 auto it = functionStates.find(funcName);
694 if (it == functionStates.end())
703 functionStates[funcName] = inlineState;
704 worklist.push_back({funcName, searchpath});
707 "considering extern function %s at %d for inlining",
708 funcName.data(), subThreshold);
710 else if (!it->second.inlined &&
711 (!it->second.processed || it->second.allowReconsidering) &&
712 it->second.costLimit < subThreshold)
719 if (it->second.processed)
722 "reconsidering extern function %s at %d for inlining, increasing from %d",
723 funcName.data(), subThreshold, it->second.costLimit);
725 it->second.processed =
false;
726 it->second.allowReconsidering =
false;
727 worklist.push_back({funcName, searchpath});
729 it->second.costLimit = subThreshold;
735 if (referencedFunction->isInterposable())
748 "cannot inline %s due to required function %s not being inlinable",
749 F.getName().data(), referencedFunction->getName().data());
754 importVars.insert(referencedFunction->getName());
757 for (
auto& recImportVar : recImportVars)
758 importVars.insert(recImportVar.first());
768 static std::unique_ptr<llvm::ModuleSummaryIndex>
771 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
772 llvm::MemoryBuffer::getFile(path);
774 if (std::error_code EC = MBOrErr.getError())
776 ilog(
DEBUG1,
"failed to open %s: %s", path.data(),
777 EC.message().c_str());
781 llvm::MemoryBufferRef ref(*MBOrErr.get().get());
783 #if LLVM_VERSION_MAJOR > 3
784 llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
785 llvm::getModuleSummaryIndex(ref);
787 return std::move(IndexOrErr.get());
788 elog(
FATAL,
"failed to load summary \"%s\": %s",
790 toString(IndexOrErr.takeError()).c_str());
792 llvm::ErrorOr<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
793 llvm::getModuleSummaryIndex(ref, [](
const llvm::DiagnosticInfo &) {});
795 return std::move(IndexOrErr.get());
796 elog(
FATAL,
"failed to load summary \"%s\": %s",
798 IndexOrErr.getError().message().c_str());
811 if (!modpath.startswith(
"$libdir/"))
829 searchpath.push_back(it->second.get());
836 static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
839 llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
841 for (
auto index : path)
843 #if LLVM_VERSION_MAJOR > 4
844 llvm::ValueInfo funcVI =
index->getValueInfo(guid);
848 for (
auto &gv : funcVI.getSummaryList())
849 matches.push_back(gv.get());
851 const llvm::const_gvsummary_iterator &
I =
852 index->findGlobalValueSummaryList(guid);
855 for (
auto &gv :
I->second)
856 matches.push_back(gv.get());
867 static llvm::Function*
870 llvm::StringRef
Name)
872 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
874 llvm::LLVMContext &Context =
F->getContext();
875 llvm::IRBuilder<> Builder(Context);
877 llvm::BasicBlock *BB;
878 llvm::CallInst *fwdcall;
879 #if LLVM_VERSION_MAJOR < 14
880 llvm::Attribute inlineAttribute;
883 AF = llvm::Function::Create(
F->getFunctionType(),
884 LinkageTypes::AvailableExternallyLinkage,
885 Name, importMod.get());
886 BB = llvm::BasicBlock::Create(Context,
"entry", AF);
888 Builder.SetInsertPoint(BB);
889 fwdcall = Builder.CreateCall(
F, &*AF->arg_begin());
890 #if LLVM_VERSION_MAJOR < 14
891 inlineAttribute = llvm::Attribute::get(Context,
892 llvm::Attribute::AlwaysInline);
893 fwdcall->addAttribute(~0U, inlineAttribute);
895 fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
897 Builder.CreateRet(fwdcall);
elog(ERROR, "%s: %s", p2, msg)
char pkglib_path[MAXPGPATH]
Assert(fmt[strlen(fmt) - 1] !='\n')
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
void llvm_inline(LLVMModuleRef M)
llvm::SmallVector< InlineWorkListItem, 128 > InlineWorkList
static std::unique_ptr< ImportMapTy > llvm_build_inline_plan(llvm::Module *mod)
const int inline_initial_cost
static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary(llvm::StringRef path)
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > InlineSearchPath
static llvm::Module * load_module_cached(llvm::StringRef modPath)
struct FunctionInlineState FunctionInlineState
static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid(const InlineSearchPath &path, llvm::GlobalValue::GUID guid)
const float inline_cost_decay_factor
llvm::StringMap< llvm::StringSet<> > ImportMapTy
static std::unique_ptr< llvm::Module > load_module(llvm::StringRef Identifier)
struct InlineWorkListItem InlineWorkListItem
static void function_references(llvm::Function &F, int &running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &referencedFunctions)
static void add_module_to_inline_search_path(InlineSearchPath &path, llvm::StringRef modpath)
llvm::StringMap< FunctionInlineState > FunctionInlineStates
static llvm::Function * create_redirection_function(std::unique_ptr< llvm::Module > &importMod, llvm::Function *F, llvm::StringRef Name)
llvm::StringMap< std::unique_ptr< llvm::ModuleSummaryIndex > > SummaryCache
llvm::StringMap< std::unique_ptr< llvm::Module > > ModuleCache
llvm::ManagedStatic< ModuleCache > module_cache
llvm::ManagedStatic< SummaryCache > summary_cache
static void llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
static bool function_inlinable(llvm::Function &F, int threshold, FunctionInlineStates &functionState, InlineWorkList &worklist, InlineSearchPath &searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &visitedFunctions, int &running_instcount, llvm::StringSet<> &importVars)
llvm::StringRef symbolName
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > searchpath