34 #include <sys/types.h>
42 #include <llvm-c/Core.h>
43 #include <llvm-c/BitReader.h>
48 #include <llvm/ADT/SetVector.h>
49 #include <llvm/ADT/StringSet.h>
50 #include <llvm/ADT/StringMap.h>
51 #include <llvm/Analysis/ModuleSummaryAnalysis.h>
52 #if LLVM_VERSION_MAJOR > 3
53 #include <llvm/Bitcode/BitcodeReader.h>
55 #include <llvm/Bitcode/ReaderWriter.h>
56 #include <llvm/Support/Error.h>
58 #include <llvm/IR/Attributes.h>
59 #include <llvm/IR/DebugInfo.h>
60 #include <llvm/IR/IntrinsicInst.h>
61 #include <llvm/IR/IRBuilder.h>
62 #include <llvm/IR/ModuleSummaryIndex.h>
63 #include <llvm/Linker/IRMover.h>
64 #include <llvm/Support/ManagedStatic.h>
65 #include <llvm/Support/MemoryBuffer.h>
80 llvm::SmallVector<llvm::ModuleSummaryIndex *, 2>
searchpath;
111 typedef llvm::StringMap<std::unique_ptr<llvm::Module> >
ModuleCache;
113 typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> >
SummaryCache;
122 static std::unique_ptr<llvm::Module>
load_module(LLVMContextRef
c, llvm::StringRef Identifier);
123 static std::unique_ptr<llvm::ModuleSummaryIndex>
llvm_load_summary(llvm::StringRef path);
128 llvm::StringRef
Name);
135 llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
136 int &running_instcount,
137 llvm::StringSet<> &importVars);
139 int &running_instcount,
140 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
141 llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions);
144 static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
152 #define ilog(...) (void) 0
174 LLVMContextRef lc = LLVMGetModuleContext(M);
175 llvm::Module *mod = llvm::unwrap(M);
178 if (!globalsToInline)
187 static std::unique_ptr<ImportMapTy>
190 std::unique_ptr<ImportMapTy> globalsToInline(
new ImportMapTy());
199 if (defaultSearchPath.empty())
208 for (
const llvm::Function &funcDecl : mod->functions())
214 if (!funcDecl.isDeclaration())
218 if (funcDecl.isIntrinsic())
223 worklist.push_back(item);
228 functionStates[funcDecl.getName()] = inlineState;
235 while (!worklist.empty())
242 llvm::GlobalValue::GUID funcGUID;
246 funcGUID = llvm::GlobalValue::getGUID(cfuncname);
263 const llvm::FunctionSummary *fs;
264 llvm::StringRef modPath = gvs->modulePath();
265 llvm::Module *defMod;
266 llvm::Function *funcDef;
268 fs = llvm::cast<llvm::FunctionSummary>(gvs);
270 #if LLVM_VERSION_MAJOR > 3
271 if (gvs->notEligibleToImport())
273 ilog(
DEBUG1,
"ineligibile to import %s due to summary",
279 if ((
int) fs->instCount() > inlineState.
costLimit)
281 ilog(
DEBUG1,
"ineligibile to import %s due to early threshold: %u vs %u",
282 symbolName.data(), fs->instCount(), inlineState.
costLimit);
288 if (defMod->materializeMetadata())
289 elog(
FATAL,
"failed to materialize metadata");
291 funcDef = defMod->getFunction(cfuncname);
300 if (funcDef->materialize())
301 elog(
FATAL,
"failed to materialize metadata");
303 Assert(!funcDef->isDeclaration());
304 Assert(funcDef->hasExternalLinkage());
306 llvm::StringSet<> importVars;
307 llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
308 int running_instcount = 0;
330 if (running_instcount > inlineState.
costLimit)
332 ilog(
DEBUG1,
"skipping inlining of %s due to late threshold %d vs %d",
333 symbolName.data(), running_instcount, inlineState.
costLimit);
338 ilog(
DEBUG1,
"inline top function %s total_instcount: %d, partial: %d",
339 symbolName.data(), running_instcount, fs->instCount());
342 importVars.insert(symbolName);
345 llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
346 for (
auto& importVar : importVars)
347 modGlobalsToInline.insert(importVar.first());
348 Assert(modGlobalsToInline.size() > 0);
376 return globalsToInline;
386 llvm::IRMover Mover(*mod);
388 for (
const auto& toInline : *globalsToInline)
390 const llvm::StringRef& modPath = toInline.first();
391 const llvm::StringSet<>& modGlobalsToInline = toInline.second;
392 llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
395 std::unique_ptr<llvm::Module> importMod(std::move((*
module_cache)[modPath]));
398 if (modGlobalsToInline.empty())
401 for (
auto &glob: modGlobalsToInline)
403 llvm::StringRef SymbolName = glob.first();
409 llvm::GlobalValue *valueToImport = importMod->getNamedValue(
funcname);
412 elog(
FATAL,
"didn't refind value %s to import", SymbolName.data());
420 if (llvm::isa<llvm::Function>(valueToImport))
422 llvm::Function *
F = llvm::dyn_cast<llvm::Function>(valueToImport);
423 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
429 llvm::stripDebugInfo(*
F);
445 GlobalsToImport.insert(AF);
446 llvm::stripDebugInfo(*AF);
449 if (valueToImport->hasExternalLinkage())
451 valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
455 GlobalsToImport.insert(valueToImport);
457 modPath.data(), SymbolName.data());
461 #if LLVM_VERSION_MAJOR > 4
462 #define IRMOVE_PARAMS , false
463 #elif LLVM_VERSION_MAJOR > 3
464 #define IRMOVE_PARAMS , false, false
466 #define IRMOVE_PARAMS
468 if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
469 [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}
471 elog(
FATAL,
"function import failed with linker error");
488 std::make_pair(modPath,
load_module(lc, modPath))).first;
491 return it->second.get();
494 static std::unique_ptr<llvm::Module>
497 LLVMMemoryBufferRef
buf;
504 if (LLVMCreateMemoryBufferWithContentsOfFile(path, &
buf, &msg))
505 elog(
FATAL,
"failed to open bitcode file \"%s\": %s",
507 if (LLVMGetBitcodeModuleInContext2(lc,
buf, &mod))
508 elog(
FATAL,
"failed to parse bitcode in file \"%s\"", path);
515 llvm::StripDebugInfo(*llvm::unwrap(mod));
517 return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
526 int &running_instcount,
527 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
528 llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions)
530 llvm::SmallPtrSet<const llvm::User *, 32> Visited;
532 for (llvm::BasicBlock &BB :
F)
534 for (llvm::Instruction &
I : BB)
536 if (llvm::isa<llvm::DbgInfoIntrinsic>(
I))
539 llvm::SmallVector<llvm::User *, 8> Worklist;
540 Worklist.push_back(&
I);
544 while (!Worklist.empty()) {
545 llvm::User *U = Worklist.pop_back_val();
548 if (!Visited.insert(U).second)
551 for (
auto &OI : U->operands()) {
552 llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
555 if (llvm::isa<llvm::BlockAddress>(Operand))
557 if (
auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
558 referencedVars.insert(GV);
559 if (GV->hasInitializer())
560 Worklist.push_back(GV->getInitializer());
563 if (
auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
564 referencedFunctions.insert(CF);
567 Worklist.push_back(Operand);
587 llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
588 int &running_instcount,
589 llvm::StringSet<> &importVars)
592 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
593 llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
596 if (
F.isInterposable())
603 if (
F.hasAvailableExternallyLinkage())
606 ilog(
DEBUG1,
"checking inlinability of %s",
F.getName().data());
609 elog(
FATAL,
"failed to materialize metadata");
611 #if LLVM_VERSION_MAJOR < 14
612 #define hasFnAttr hasFnAttribute
615 if (
F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
617 ilog(
DEBUG1,
"ineligibile to import %s due to noinline",
624 for (llvm::GlobalVariable* rv: referencedVars)
626 if (rv->materialize())
627 elog(
FATAL,
"failed to materialize metadata");
633 if (rv->isThreadLocal())
635 ilog(
DEBUG1,
"cannot inline %s due to thread-local variable %s",
636 F.getName().data(), rv->getName().data());
644 if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
655 if (!rv->isConstant())
657 ilog(
DEBUG1,
"cannot inline %s due to uncloneable variable %s",
658 F.getName().data(), rv->getName().data());
662 ilog(
DEBUG1,
"memorizing global var %s linkage %d for inlining",
663 rv->getName().data(), (
int)rv->getLinkage());
665 importVars.insert(rv->getName());
667 running_instcount += 5;
670 visitedFunctions.insert(&
F);
676 for (llvm::Function* referencedFunction: referencedFunctions)
678 llvm::StringSet<> recImportVars;
680 if (referencedFunction->materialize())
681 elog(
FATAL,
"failed to materialize metadata");
683 if (referencedFunction->isIntrinsic())
687 if (!visitedFunctions.insert(referencedFunction).second)
695 if (referencedFunction->hasExternalLinkage())
697 llvm::StringRef funcName = referencedFunction->getName();
703 if (subThreshold < 5)
706 auto it = functionStates.find(funcName);
707 if (it == functionStates.end())
716 functionStates[funcName] = inlineState;
717 worklist.push_back({funcName, searchpath});
720 "considering extern function %s at %d for inlining",
721 funcName.data(), subThreshold);
723 else if (!it->second.inlined &&
724 (!it->second.processed || it->second.allowReconsidering) &&
725 it->second.costLimit < subThreshold)
732 if (it->second.processed)
735 "reconsidering extern function %s at %d for inlining, increasing from %d",
736 funcName.data(), subThreshold, it->second.costLimit);
738 it->second.processed =
false;
739 it->second.allowReconsidering =
false;
740 worklist.push_back({funcName, searchpath});
742 it->second.costLimit = subThreshold;
748 if (referencedFunction->isInterposable())
761 "cannot inline %s due to required function %s not being inlinable",
762 F.getName().data(), referencedFunction->getName().data());
767 importVars.insert(referencedFunction->getName());
770 for (
auto& recImportVar : recImportVars)
771 importVars.insert(recImportVar.first());
781 static std::unique_ptr<llvm::ModuleSummaryIndex>
784 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
785 llvm::MemoryBuffer::getFile(path);
787 if (std::error_code EC = MBOrErr.getError())
789 ilog(
DEBUG1,
"failed to open %s: %s", path.data(),
790 EC.message().c_str());
794 llvm::MemoryBufferRef ref(*MBOrErr.get().get());
796 #if LLVM_VERSION_MAJOR > 3
797 llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
798 llvm::getModuleSummaryIndex(ref);
800 return std::move(IndexOrErr.get());
801 elog(
FATAL,
"failed to load summary \"%s\": %s",
803 toString(IndexOrErr.takeError()).c_str());
805 llvm::ErrorOr<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
806 llvm::getModuleSummaryIndex(ref, [](
const llvm::DiagnosticInfo &) {});
808 return std::move(IndexOrErr.get());
809 elog(
FATAL,
"failed to load summary \"%s\": %s",
811 IndexOrErr.getError().message().c_str());
824 if (!modpath.startswith(
"$libdir/"))
842 searchpath.push_back(it->second.get());
849 static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
852 llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
854 for (
auto index : path)
856 #if LLVM_VERSION_MAJOR > 4
857 llvm::ValueInfo funcVI =
index->getValueInfo(guid);
861 for (
auto &gv : funcVI.getSummaryList())
862 matches.push_back(gv.get());
864 const llvm::const_gvsummary_iterator &
I =
865 index->findGlobalValueSummaryList(guid);
868 for (
auto &gv :
I->second)
869 matches.push_back(gv.get());
880 static llvm::Function*
883 llvm::StringRef
Name)
885 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
887 llvm::LLVMContext &Context =
F->getContext();
888 llvm::IRBuilder<> Builder(Context);
890 llvm::BasicBlock *BB;
891 llvm::CallInst *fwdcall;
892 #if LLVM_VERSION_MAJOR < 14
893 llvm::Attribute inlineAttribute;
896 AF = llvm::Function::Create(
F->getFunctionType(),
897 LinkageTypes::AvailableExternallyLinkage,
898 Name, importMod.get());
899 BB = llvm::BasicBlock::Create(Context,
"entry", AF);
901 Builder.SetInsertPoint(BB);
902 fwdcall = Builder.CreateCall(
F, &*AF->arg_begin());
903 #if LLVM_VERSION_MAJOR < 14
904 inlineAttribute = llvm::Attribute::get(Context,
905 llvm::Attribute::AlwaysInline);
906 fwdcall->addAttribute(~0U, inlineAttribute);
908 fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
910 Builder.CreateRet(fwdcall);
elog(ERROR, "%s: %s", p2, msg)
char pkglib_path[MAXPGPATH]
Assert(fmt[strlen(fmt) - 1] !='\n')
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
void llvm_inline(LLVMModuleRef M)
llvm::SmallVector< InlineWorkListItem, 128 > InlineWorkList
const int inline_initial_cost
static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary(llvm::StringRef path)
void llvm_inline_reset_caches(void)
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > InlineSearchPath
struct FunctionInlineState FunctionInlineState
static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid(const InlineSearchPath &path, llvm::GlobalValue::GUID guid)
const float inline_cost_decay_factor
llvm::StringMap< llvm::StringSet<> > ImportMapTy
static std::unique_ptr< llvm::Module > load_module(LLVMContextRef c, llvm::StringRef Identifier)
struct InlineWorkListItem InlineWorkListItem
static void function_references(llvm::Function &F, int &running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &referencedFunctions)
static void add_module_to_inline_search_path(InlineSearchPath &path, llvm::StringRef modpath)
static llvm::Module * load_module_cached(LLVMContextRef c, llvm::StringRef modPath)
llvm::StringMap< FunctionInlineState > FunctionInlineStates
static llvm::Function * create_redirection_function(std::unique_ptr< llvm::Module > &importMod, llvm::Function *F, llvm::StringRef Name)
llvm::StringMap< std::unique_ptr< llvm::ModuleSummaryIndex > > SummaryCache
llvm::StringMap< std::unique_ptr< llvm::Module > > ModuleCache
llvm::ManagedStatic< ModuleCache > module_cache
llvm::ManagedStatic< SummaryCache > summary_cache
static std::unique_ptr< ImportMapTy > llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod)
static void llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
static bool function_inlinable(llvm::Function &F, int threshold, FunctionInlineStates &functionState, InlineWorkList &worklist, InlineSearchPath &searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &visitedFunctions, int &running_instcount, llvm::StringSet<> &importVars)
llvm::StringRef symbolName
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > searchpath