PostgreSQL Source Code  git master
llvmjit_inline.cpp File Reference
#include "postgres.h"
#include "jit/llvmjit.h"
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "common/string.h"
#include "miscadmin.h"
#include "storage/fd.h"
#include <llvm-c/Core.h>
#include <llvm-c/BitReader.h>
#include <llvm/ADT/SetVector.h>
#include <llvm/ADT/StringSet.h>
#include <llvm/ADT/StringMap.h>
#include <llvm/Analysis/ModuleSummaryAnalysis.h>
#include <llvm/Bitcode/BitcodeReader.h>
#include <llvm/IR/Attributes.h>
#include <llvm/IR/DebugInfo.h>
#include <llvm/IR/IntrinsicInst.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/ModuleSummaryIndex.h>
#include <llvm/Linker/IRMover.h>
#include <llvm/Support/ManagedStatic.h>
#include <llvm/Support/MemoryBuffer.h>
Include dependency graph for llvmjit_inline.cpp:

Go to the source code of this file.

Data Structures

struct  InlineWorkListItem
 
struct  FunctionInlineState
 

Macros

#define ilog(...)   (void) 0
 
#define hasFnAttr   hasFnAttribute
 
#define starts_with   startswith
 

Typedefs

typedef llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > InlineSearchPath
 
typedef struct InlineWorkListItem InlineWorkListItem
 
typedef llvm::SmallVector< InlineWorkListItem, 128 > InlineWorkList
 
typedef struct FunctionInlineState FunctionInlineState
 
typedef llvm::StringMap< FunctionInlineStateFunctionInlineStates
 
typedef llvm::StringMap< llvm::StringSet<> > ImportMapTy
 
typedef llvm::StringMap< std::unique_ptr< llvm::Module > > ModuleCache
 
typedef llvm::StringMap< std::unique_ptr< llvm::ModuleSummaryIndex > > SummaryCache
 

Functions

static std::unique_ptr< ImportMapTyllvm_build_inline_plan (LLVMContextRef lc, llvm::Module *mod)
 
static void llvm_execute_inline_plan (llvm::Module *mod, ImportMapTy *globalsToInline)
 
static llvm::Module * load_module_cached (LLVMContextRef c, llvm::StringRef modPath)
 
static std::unique_ptr< llvm::Module > load_module (LLVMContextRef c, llvm::StringRef Identifier)
 
static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary (llvm::StringRef path)
 
static llvm::Function * create_redirection_function (std::unique_ptr< llvm::Module > &importMod, llvm::Function *F, llvm::StringRef Name)
 
static bool function_inlinable (llvm::Function &F, int threshold, FunctionInlineStates &functionState, InlineWorkList &worklist, InlineSearchPath &searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &visitedFunctions, int &running_instcount, llvm::StringSet<> &importVars)
 
static void function_references (llvm::Function &F, int &running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &referencedFunctions)
 
static void add_module_to_inline_search_path (InlineSearchPath &path, llvm::StringRef modpath)
 
static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid (const InlineSearchPath &path, llvm::GlobalValue::GUID guid)
 
void llvm_inline_reset_caches (void)
 
void llvm_inline (LLVMModuleRef M)
 

Variables

const float inline_cost_decay_factor = 0.5
 
const int inline_initial_cost = 150
 
llvm::ManagedStatic< ModuleCachemodule_cache
 
llvm::ManagedStatic< SummaryCachesummary_cache
 

Macro Definition Documentation

◆ hasFnAttr

#define hasFnAttr   hasFnAttribute

◆ ilog

#define ilog (   ...)    (void) 0

Definition at line 147 of file llvmjit_inline.cpp.

◆ starts_with

#define starts_with   startswith

Typedef Documentation

◆ FunctionInlineState

◆ FunctionInlineStates

typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates

Definition at line 90 of file llvmjit_inline.cpp.

◆ ImportMapTy

typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy

Definition at line 96 of file llvmjit_inline.cpp.

◆ InlineSearchPath

typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath

Definition at line 67 of file llvmjit_inline.cpp.

◆ InlineWorkList

typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList

Definition at line 77 of file llvmjit_inline.cpp.

◆ InlineWorkListItem

◆ ModuleCache

typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache

Definition at line 106 of file llvmjit_inline.cpp.

◆ SummaryCache

typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache

Definition at line 108 of file llvmjit_inline.cpp.

Function Documentation

◆ add_module_to_inline_search_path()

static void add_module_to_inline_search_path ( InlineSearchPath path,
llvm::StringRef  modpath 
)
static

Definition at line 797 of file llvmjit_inline.cpp.

798 {
799  /* only extension in libdir are candidates for inlining for now */
800 #if LLVM_VERSION_MAJOR < 16
801 #define starts_with startswith
802 #endif
803  if (!modpath.starts_with("$libdir/"))
804  return;
805 
806  /* if there's no match, attempt to load */
807  auto it = summary_cache->find(modpath);
808  if (it == summary_cache->end())
809  {
810  std::string path(modpath);
811  path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
812  path += ".index.bc";
813  (*summary_cache)[modpath] = llvm_load_summary(path);
814  it = summary_cache->find(modpath);
815  }
816 
817  Assert(it != summary_cache->end());
818 
819  /* if the entry isn't NULL, it's validly loaded */
820  if (it->second)
821  searchpath.push_back(it->second.get());
822 }
#define Assert(condition)
Definition: c.h:858
char pkglib_path[MAXPGPATH]
Definition: globals.c:79
static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary(llvm::StringRef path)
llvm::ManagedStatic< SummaryCache > summary_cache
char string[11]
Definition: preproc-type.c:52

References Assert, llvm_load_summary(), pkglib_path, and summary_cache.

Referenced by llvm_build_inline_plan().

◆ create_redirection_function()

static llvm::Function * create_redirection_function ( std::unique_ptr< llvm::Module > &  importMod,
llvm::Function *  F,
llvm::StringRef  Name 
)
static

Definition at line 850 of file llvmjit_inline.cpp.

853 {
854  typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
855 
856  llvm::LLVMContext &Context = F->getContext();
857  llvm::IRBuilder<> Builder(Context);
858  llvm::Function *AF;
859  llvm::BasicBlock *BB;
860  llvm::CallInst *fwdcall;
861 #if LLVM_VERSION_MAJOR < 14
862  llvm::Attribute inlineAttribute;
863 #endif
864 
865  AF = llvm::Function::Create(F->getFunctionType(),
866  LinkageTypes::AvailableExternallyLinkage,
867  Name, importMod.get());
868  BB = llvm::BasicBlock::Create(Context, "entry", AF);
869 
870  Builder.SetInsertPoint(BB);
871  fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
872 #if LLVM_VERSION_MAJOR < 14
873  inlineAttribute = llvm::Attribute::get(Context,
874  llvm::Attribute::AlwaysInline);
875  fwdcall->addAttribute(~0U, inlineAttribute);
876 #else
877  fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
878 #endif
879  Builder.CreateRet(fwdcall);
880 
881  return AF;
882 }
#define F(X, Y, Z)
Definition: md5.c:60
Definition: c.h:741

References F.

Referenced by llvm_execute_inline_plan().

◆ function_inlinable()

static bool function_inlinable ( llvm::Function &  F,
int  threshold,
FunctionInlineStates functionState,
InlineWorkList worklist,
InlineSearchPath searchpath,
llvm::SmallPtrSet< const llvm::Function *, 8 > &  visitedFunctions,
int &  running_instcount,
llvm::StringSet<> &  importVars 
)
static

Definition at line 568 of file llvmjit_inline.cpp.

576 {
577  int subThreshold = threshold * inline_cost_decay_factor;
578  llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
579  llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
580 
581  /* can't rely on what may be inlined */
582  if (F.isInterposable())
583  return false;
584 
585  /*
586  * Can't rely on function being present. Alternatively we could create a
587  * static version of these functions?
588  */
589  if (F.hasAvailableExternallyLinkage())
590  return false;
591 
592  ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
593 
594  if (F.materialize())
595  elog(FATAL, "failed to materialize metadata");
596 
597 #if LLVM_VERSION_MAJOR < 14
598 #define hasFnAttr hasFnAttribute
599 #endif
600 
601  if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
602  {
603  ilog(DEBUG1, "ineligibile to import %s due to noinline",
604  F.getName().data());
605  return false;
606  }
607 
608  function_references(F, running_instcount, referencedVars, referencedFunctions);
609 
610  for (llvm::GlobalVariable* rv: referencedVars)
611  {
612  if (rv->materialize())
613  elog(FATAL, "failed to materialize metadata");
614 
615  /*
616  * Don't inline functions that access thread local variables. That
617  * doesn't work on current LLVM releases (but might in future).
618  */
619  if (rv->isThreadLocal())
620  {
621  ilog(DEBUG1, "cannot inline %s due to thread-local variable %s",
622  F.getName().data(), rv->getName().data());
623  return false;
624  }
625 
626  /*
627  * Never want to inline externally visible vars, cheap enough to
628  * reference.
629  */
630  if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
631  continue;
632 
633  /*
634  * If variable is file-local, we need to inline it, to be able to
635  * inline the function itself. Can't do that if the variable can be
636  * modified, because they'd obviously get out of sync.
637  *
638  * XXX: Currently not a problem, but there'd be problems with
639  * nontrivial initializers if they were allowed for postgres.
640  */
641  if (!rv->isConstant())
642  {
643  ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
644  F.getName().data(), rv->getName().data());
645  return false;
646  }
647 
648  ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
649  rv->getName().data(), (int)rv->getLinkage());
650 
651  importVars.insert(rv->getName());
652  /* small cost attributed to each cloned global */
653  running_instcount += 5;
654  }
655 
656  visitedFunctions.insert(&F);
657 
658  /*
659  * Check referenced functions. Check whether used static ones are
660  * inlinable, and remember external ones for inlining.
661  */
662  for (llvm::Function* referencedFunction: referencedFunctions)
663  {
664  llvm::StringSet<> recImportVars;
665 
666  if (referencedFunction->materialize())
667  elog(FATAL, "failed to materialize metadata");
668 
669  if (referencedFunction->isIntrinsic())
670  continue;
671 
672  /* if already visited skip, otherwise remember */
673  if (!visitedFunctions.insert(referencedFunction).second)
674  continue;
675 
676  /*
677  * We don't inline external functions directly here, instead we put
678  * them on the worklist if appropriate and check them from
679  * llvm_build_inline_plan().
680  */
681  if (referencedFunction->hasExternalLinkage())
682  {
683  llvm::StringRef funcName = referencedFunction->getName();
684 
685  /*
686  * Don't bother checking for inlining if remaining cost budget is
687  * very small.
688  */
689  if (subThreshold < 5)
690  continue;
691 
692  auto it = functionStates.find(funcName);
693  if (it == functionStates.end())
694  {
695  FunctionInlineState inlineState;
696 
697  inlineState.costLimit = subThreshold;
698  inlineState.processed = false;
699  inlineState.inlined = false;
700  inlineState.allowReconsidering = false;
701 
702  functionStates[funcName] = inlineState;
703  worklist.push_back({funcName, searchpath});
704 
705  ilog(DEBUG1,
706  "considering extern function %s at %d for inlining",
707  funcName.data(), subThreshold);
708  }
709  else if (!it->second.inlined &&
710  (!it->second.processed || it->second.allowReconsidering) &&
711  it->second.costLimit < subThreshold)
712  {
713  /*
714  * Update inlining threshold if higher. Need to re-queue
715  * to be processed if already processed with lower
716  * threshold.
717  */
718  if (it->second.processed)
719  {
720  ilog(DEBUG1,
721  "reconsidering extern function %s at %d for inlining, increasing from %d",
722  funcName.data(), subThreshold, it->second.costLimit);
723 
724  it->second.processed = false;
725  it->second.allowReconsidering = false;
726  worklist.push_back({funcName, searchpath});
727  }
728  it->second.costLimit = subThreshold;
729  }
730  continue;
731  }
732 
733  /* can't rely on what may be inlined */
734  if (referencedFunction->isInterposable())
735  return false;
736 
737  if (!function_inlinable(*referencedFunction,
738  subThreshold,
739  functionStates,
740  worklist,
741  searchpath,
742  visitedFunctions,
743  running_instcount,
744  recImportVars))
745  {
746  ilog(DEBUG1,
747  "cannot inline %s due to required function %s not being inlinable",
748  F.getName().data(), referencedFunction->getName().data());
749  return false;
750  }
751 
752  /* import referenced function itself */
753  importVars.insert(referencedFunction->getName());
754 
755  /* import referenced function and its dependents */
756  for (auto& recImportVar : recImportVars)
757  importVars.insert(recImportVar.first());
758  }
759 
760  return true;
761 }
#define FATAL
Definition: elog.h:41
#define DEBUG1
Definition: elog.h:30
#define elog(elevel,...)
Definition: elog.h:224
const float inline_cost_decay_factor
static void function_references(llvm::Function &F, int &running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &referencedFunctions)
#define ilog(...)
static bool function_inlinable(llvm::Function &F, int threshold, FunctionInlineStates &functionState, InlineWorkList &worklist, InlineSearchPath &searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &visitedFunctions, int &running_instcount, llvm::StringSet<> &importVars)

References FunctionInlineState::allowReconsidering, FunctionInlineState::costLimit, DEBUG1, elog, F, FATAL, function_references(), ilog, inline_cost_decay_factor, FunctionInlineState::inlined, and FunctionInlineState::processed.

Referenced by llvm_build_inline_plan().

◆ function_references()

static void function_references ( llvm::Function &  F,
int &  running_instcount,
llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &  referencedVars,
llvm::SmallPtrSet< llvm::Function *, 8 > &  referencedFunctions 
)
static

Definition at line 511 of file llvmjit_inline.cpp.

515 {
516  llvm::SmallPtrSet<const llvm::User *, 32> Visited;
517 
518  for (llvm::BasicBlock &BB : F)
519  {
520  for (llvm::Instruction &I : BB)
521  {
522  if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
523  continue;
524 
525  llvm::SmallVector<llvm::User *, 8> Worklist;
526  Worklist.push_back(&I);
527 
528  running_instcount++;
529 
530  while (!Worklist.empty()) {
531  llvm::User *U = Worklist.pop_back_val();
532 
533  /* visited before */
534  if (!Visited.insert(U).second)
535  continue;
536 
537  for (auto &OI : U->operands()) {
538  llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
539  if (!Operand)
540  continue;
541  if (llvm::isa<llvm::BlockAddress>(Operand))
542  continue;
543  if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
544  referencedVars.insert(GV);
545  if (GV->hasInitializer())
546  Worklist.push_back(GV->getInitializer());
547  continue;
548  }
549  if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
550  referencedFunctions.insert(CF);
551  continue;
552  }
553  Worklist.push_back(Operand);
554  }
555  }
556  }
557  }
558 }
#define I(X, Y, Z)
Definition: md5.c:63

References F, and I.

Referenced by function_inlinable().

◆ llvm_build_inline_plan()

static std::unique_ptr< ImportMapTy > llvm_build_inline_plan ( LLVMContextRef  lc,
llvm::Module *  mod 
)
static

Definition at line 183 of file llvmjit_inline.cpp.

184 {
185  std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
186  FunctionInlineStates functionStates;
187  InlineWorkList worklist;
188 
189  InlineSearchPath defaultSearchPath;
190 
191  /* attempt to add module to search path */
192  add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
193  /* if postgres isn't available, no point continuing */
194  if (defaultSearchPath.empty())
195  return nullptr;
196 
197  /*
198  * Start inlining with current references to external functions by putting
199  * them on the inlining worklist. If, during inlining of those, new extern
200  * functions need to be inlined, they'll also be put there, with a lower
201  * priority.
202  */
203  for (const llvm::Function &funcDecl : mod->functions())
204  {
205  InlineWorkListItem item = {};
206  FunctionInlineState inlineState = {};
207 
208  /* already has a definition */
209  if (!funcDecl.isDeclaration())
210  continue;
211 
212  /* llvm provides implementation */
213  if (funcDecl.isIntrinsic())
214  continue;
215 
216  item.symbolName = funcDecl.getName();
217  item.searchpath = defaultSearchPath;
218  worklist.push_back(item);
219  inlineState.costLimit = inline_initial_cost;
220  inlineState.processed = false;
221  inlineState.inlined = false;
222  inlineState.allowReconsidering = false;
223  functionStates[funcDecl.getName()] = inlineState;
224  }
225 
226  /*
227  * Iterate over pending worklist items, look them up in index, check
228  * whether they should be inlined.
229  */
230  while (!worklist.empty())
231  {
232  InlineWorkListItem item = worklist.pop_back_val();
233  llvm::StringRef symbolName = item.symbolName;
234  char *cmodname;
235  char *cfuncname;
236  FunctionInlineState &inlineState = functionStates[symbolName];
237  llvm::GlobalValue::GUID funcGUID;
238 
239  llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
240 
241  funcGUID = llvm::GlobalValue::getGUID(cfuncname);
242 
243  /* already processed */
244  if (inlineState.processed)
245  continue;
246 
247 
248  if (cmodname)
250 
251  /*
252  * Iterate over all known definitions of function, via the index. Then
253  * look up module(s), check if function actually is defined (there
254  * could be hash conflicts).
255  */
256  for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
257  {
258  const llvm::FunctionSummary *fs;
259  llvm::StringRef modPath = gvs->modulePath();
260  llvm::Module *defMod;
261  llvm::Function *funcDef;
262 
263  fs = llvm::cast<llvm::FunctionSummary>(gvs);
264 
265  if (gvs->notEligibleToImport())
266  {
267  ilog(DEBUG1, "ineligibile to import %s due to summary",
268  symbolName.data());
269  continue;
270  }
271 
272  if ((int) fs->instCount() > inlineState.costLimit)
273  {
274  ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
275  symbolName.data(), fs->instCount(), inlineState.costLimit);
276  inlineState.allowReconsidering = true;
277  continue;
278  }
279 
280  defMod = load_module_cached(lc, modPath);
281  if (defMod->materializeMetadata())
282  elog(FATAL, "failed to materialize metadata");
283 
284  funcDef = defMod->getFunction(cfuncname);
285 
286  /*
287  * This can happen e.g. in case of a hash collision of the
288  * function's name.
289  */
290  if (!funcDef)
291  continue;
292 
293  if (funcDef->materialize())
294  elog(FATAL, "failed to materialize metadata");
295 
296  Assert(!funcDef->isDeclaration());
297  Assert(funcDef->hasExternalLinkage());
298 
299  llvm::StringSet<> importVars;
300  llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
301  int running_instcount = 0;
302 
303  /*
304  * Check whether function, and objects it depends on, are
305  * inlinable.
306  */
307  if (function_inlinable(*funcDef,
308  inlineState.costLimit,
309  functionStates,
310  worklist,
311  item.searchpath,
312  visitedFunctions,
313  running_instcount,
314  importVars))
315  {
316  /*
317  * Check whether function and all its dependencies are too
318  * big. Dependencies already counted for other functions that
319  * will get inlined are not counted again. While this make
320  * things somewhat order dependent, I can't quite see a point
321  * in a different behaviour.
322  */
323  if (running_instcount > inlineState.costLimit)
324  {
325  ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
326  symbolName.data(), running_instcount, inlineState.costLimit);
327  inlineState.allowReconsidering = true;
328  continue;
329  }
330 
331  ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
332  symbolName.data(), running_instcount, fs->instCount());
333 
334  /* import referenced function itself */
335  importVars.insert(symbolName);
336 
337  {
338  llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
339  for (auto& importVar : importVars)
340  modGlobalsToInline.insert(importVar.first());
341  Assert(modGlobalsToInline.size() > 0);
342  }
343 
344  /* mark function as inlined */
345  inlineState.inlined = true;
346 
347  /*
348  * Found definition to inline, don't look for further
349  * potential definitions.
350  */
351  break;
352  }
353  else
354  {
355  ilog(DEBUG1, "had to skip inlining %s",
356  symbolName.data());
357 
358  /* It's possible there's another definition that's inlinable. */
359  }
360  }
361 
362  /*
363  * Signal that we're done with symbol, whether successful (inlined =
364  * true above) or not.
365  */
366  inlineState.processed = true;
367  }
368 
369  return globalsToInline;
370 }
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
Definition: llvmjit.c:1152
llvm::SmallVector< InlineWorkListItem, 128 > InlineWorkList
const int inline_initial_cost
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > InlineSearchPath
static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid(const InlineSearchPath &path, llvm::GlobalValue::GUID guid)
llvm::StringMap< llvm::StringSet<> > ImportMapTy
static void add_module_to_inline_search_path(InlineSearchPath &path, llvm::StringRef modpath)
static llvm::Module * load_module_cached(LLVMContextRef c, llvm::StringRef modPath)
llvm::StringMap< FunctionInlineState > FunctionInlineStates
llvm::StringRef symbolName
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > searchpath

References add_module_to_inline_search_path(), FunctionInlineState::allowReconsidering, Assert, FunctionInlineState::costLimit, DEBUG1, elog, FATAL, function_inlinable(), ilog, inline_initial_cost, FunctionInlineState::inlined, llvm_split_symbol_name(), load_module_cached(), FunctionInlineState::processed, InlineWorkListItem::searchpath, summaries_for_guid(), and InlineWorkListItem::symbolName.

Referenced by llvm_inline().

◆ llvm_execute_inline_plan()

static void llvm_execute_inline_plan ( llvm::Module *  mod,
ImportMapTy globalsToInline 
)
static

Definition at line 377 of file llvmjit_inline.cpp.

378 {
379  llvm::IRMover Mover(*mod);
380 
381  for (const auto& toInline : *globalsToInline)
382  {
383  const llvm::StringRef& modPath = toInline.first();
384  const llvm::StringSet<>& modGlobalsToInline = toInline.second;
385  llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
386 
387  Assert(module_cache->count(modPath));
388  std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
389  module_cache->erase(modPath);
390 
391  if (modGlobalsToInline.empty())
392  continue;
393 
394  for (auto &glob: modGlobalsToInline)
395  {
396  llvm::StringRef SymbolName = glob.first();
397  char *modname;
398  char *funcname;
399 
400  llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
401 
402  llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
403 
404  if (!valueToImport)
405  elog(FATAL, "didn't refind value %s to import", SymbolName.data());
406 
407  /*
408  * For functions (global vars are only inlined if already static),
409  * mark imported variables as being clones from other
410  * functions. That a) avoids symbol conflicts b) allows the
411  * optimizer to perform inlining.
412  */
413  if (llvm::isa<llvm::Function>(valueToImport))
414  {
415  llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
416  typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
417 
418  /*
419  * Per-function info isn't necessarily stripped yet, as the
420  * module is lazy-loaded when stripped above.
421  */
422  llvm::stripDebugInfo(*F);
423 
424  /*
425  * If the to-be-imported function is one referenced including
426  * its module name, create a tiny inline function that just
427  * forwards the call. One might think a GlobalAlias would do
428  * the trick, but a) IRMover doesn't override a declaration
429  * with an alias pointing to a definition (instead renaming
430  * it), b) Aliases can't be AvailableExternally.
431  */
432  if (modname)
433  {
434  llvm::Function *AF;
435 
436  AF = create_redirection_function(importMod, F, SymbolName);
437 
438  GlobalsToImport.insert(AF);
439  llvm::stripDebugInfo(*AF);
440  }
441 
442  if (valueToImport->hasExternalLinkage())
443  {
444  valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
445  }
446  }
447 
448  GlobalsToImport.insert(valueToImport);
449  ilog(DEBUG1, "performing import of %s %s",
450  modPath.data(), SymbolName.data());
451 
452  }
453 
454  if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
455  [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {},
456  /*IsPerformingImport=*/false))
457  elog(FATAL, "function import failed with linker error");
458  }
459 }
#define funcname
Definition: indent_codes.h:69
static llvm::Function * create_redirection_function(std::unique_ptr< llvm::Module > &importMod, llvm::Function *F, llvm::StringRef Name)
llvm::ManagedStatic< ModuleCache > module_cache

References Assert, create_redirection_function(), DEBUG1, elog, F, FATAL, funcname, ilog, llvm_split_symbol_name(), and module_cache.

Referenced by llvm_inline().

◆ llvm_inline()

void llvm_inline ( LLVMModuleRef  M)

Definition at line 167 of file llvmjit_inline.cpp.

168 {
169  LLVMContextRef lc = LLVMGetModuleContext(M);
170  llvm::Module *mod = llvm::unwrap(M);
171 
172  std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(lc, mod);
173  if (!globalsToInline)
174  return;
175  llvm_execute_inline_plan(mod, globalsToInline.get());
176 }
static std::unique_ptr< ImportMapTy > llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod)
static void llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)

References llvm_build_inline_plan(), and llvm_execute_inline_plan().

Referenced by llvm_compile_module().

◆ llvm_inline_reset_caches()

void llvm_inline_reset_caches ( void  )

Definition at line 156 of file llvmjit_inline.cpp.

157 {
158  module_cache->clear();
159  summary_cache->clear();
160 }

References module_cache, and summary_cache.

Referenced by llvm_recreate_llvm_context().

◆ llvm_load_summary()

static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary ( llvm::StringRef  path)
static

Definition at line 768 of file llvmjit_inline.cpp.

769 {
770  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
771  llvm::MemoryBuffer::getFile(path);
772 
773  if (std::error_code EC = MBOrErr.getError())
774  {
775  ilog(DEBUG1, "failed to open %s: %s", path.data(),
776  EC.message().c_str());
777  }
778  else
779  {
780  llvm::MemoryBufferRef ref(*MBOrErr.get().get());
781 
782  llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
783  llvm::getModuleSummaryIndex(ref);
784  if (IndexOrErr)
785  return std::move(IndexOrErr.get());
786  elog(FATAL, "failed to load summary \"%s\": %s",
787  path.data(),
788  toString(IndexOrErr.takeError()).c_str());
789  }
790  return nullptr;
791 }

References DEBUG1, elog, FATAL, and ilog.

Referenced by add_module_to_inline_search_path().

◆ load_module()

static std::unique_ptr< llvm::Module > load_module ( LLVMContextRef  c,
llvm::StringRef  Identifier 
)
static

Definition at line 481 of file llvmjit_inline.cpp.

482 {
483  LLVMMemoryBufferRef buf;
484  LLVMModuleRef mod;
485  char path[MAXPGPATH];
486  char *msg;
487 
488  snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
489 
490  if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
491  elog(FATAL, "failed to open bitcode file \"%s\": %s",
492  path, msg);
493  if (LLVMGetBitcodeModuleInContext2(lc, buf, &mod))
494  elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
495 
496  /*
497  * Currently there's no use in more detailed debug info for JITed
498  * code. Until that changes, not much point in wasting memory and cycles
499  * on processing debuginfo.
500  */
501  llvm::StripDebugInfo(*llvm::unwrap(mod));
502 
503  return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
504 }
#define MAXPGPATH
static char * buf
Definition: pg_test_fsync.c:73
#define snprintf
Definition: port.h:238

References buf, elog, FATAL, MAXPGPATH, pkglib_path, and snprintf.

Referenced by load_module_cached().

◆ load_module_cached()

static llvm::Module * load_module_cached ( LLVMContextRef  c,
llvm::StringRef  modPath 
)
static

Definition at line 468 of file llvmjit_inline.cpp.

469 {
470  auto it = module_cache->find(modPath);
471  if (it == module_cache->end())
472  {
473  it = module_cache->insert(
474  std::make_pair(modPath, load_module(lc, modPath))).first;
475  }
476 
477  return it->second.get();
478 }
static std::unique_ptr< llvm::Module > load_module(LLVMContextRef c, llvm::StringRef Identifier)

References load_module(), and module_cache.

Referenced by llvm_build_inline_plan().

◆ summaries_for_guid()

static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid ( const InlineSearchPath path,
llvm::GlobalValue::GUID  guid 
)
static

Definition at line 829 of file llvmjit_inline.cpp.

830 {
831  llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
832 
833  for (auto index : path)
834  {
835  llvm::ValueInfo funcVI = index->getValueInfo(guid);
836 
837  /* if index doesn't know function, we don't have a body, continue */
838  if (funcVI)
839  for (auto &gv : funcVI.getSummaryList())
840  matches.push_back(gv.get());
841  }
842 
843  return matches;
844 }
Definition: type.h:95

Referenced by llvm_build_inline_plan().

Variable Documentation

◆ inline_cost_decay_factor

const float inline_cost_decay_factor = 0.5

Definition at line 99 of file llvmjit_inline.cpp.

Referenced by function_inlinable().

◆ inline_initial_cost

const int inline_initial_cost = 150

Definition at line 100 of file llvmjit_inline.cpp.

Referenced by llvm_build_inline_plan().

◆ module_cache

llvm::ManagedStatic<ModuleCache> module_cache

◆ summary_cache

llvm::ManagedStatic<SummaryCache> summary_cache