PostgreSQL Source Code  git master
llvmjit_inline.cpp File Reference
#include "postgres.h"
#include "jit/llvmjit.h"
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "common/string.h"
#include "miscadmin.h"
#include "storage/fd.h"
#include <llvm-c/Core.h>
#include <llvm-c/BitReader.h>
#include <llvm/ADT/SetVector.h>
#include <llvm/ADT/StringSet.h>
#include <llvm/ADT/StringMap.h>
#include <llvm/Analysis/ModuleSummaryAnalysis.h>
#include <llvm/Bitcode/ReaderWriter.h>
#include <llvm/Support/Error.h>
#include <llvm/IR/Attributes.h>
#include <llvm/IR/DebugInfo.h>
#include <llvm/IR/IntrinsicInst.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/ModuleSummaryIndex.h>
#include <llvm/Linker/IRMover.h>
#include <llvm/Support/ManagedStatic.h>
#include <llvm/Support/MemoryBuffer.h>
Include dependency graph for llvmjit_inline.cpp:

Go to the source code of this file.

Data Structures

struct  InlineWorkListItem
 
struct  FunctionInlineState
 

Macros

#define ilog(...)   (void) 0
 
#define IRMOVE_PARAMS
 
#define hasFnAttr   hasFnAttribute
 

Typedefs

typedef llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > InlineSearchPath
 
typedef struct InlineWorkListItem InlineWorkListItem
 
typedef llvm::SmallVector< InlineWorkListItem, 128 > InlineWorkList
 
typedef struct FunctionInlineState FunctionInlineState
 
typedef llvm::StringMap< FunctionInlineStateFunctionInlineStates
 
typedef llvm::StringMap< llvm::StringSet<> > ImportMapTy
 
typedef llvm::StringMap< std::unique_ptr< llvm::Module > > ModuleCache
 
typedef llvm::StringMap< std::unique_ptr< llvm::ModuleSummaryIndex > > SummaryCache
 

Functions

static std::unique_ptr< ImportMapTyllvm_build_inline_plan (llvm::Module *mod)
 
static void llvm_execute_inline_plan (llvm::Module *mod, ImportMapTy *globalsToInline)
 
static llvm::Module * load_module_cached (llvm::StringRef modPath)
 
static std::unique_ptr< llvm::Module > load_module (llvm::StringRef Identifier)
 
static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary (llvm::StringRef path)
 
static llvm::Function * create_redirection_function (std::unique_ptr< llvm::Module > &importMod, llvm::Function *F, llvm::StringRef Name)
 
static bool function_inlinable (llvm::Function &F, int threshold, FunctionInlineStates &functionState, InlineWorkList &worklist, InlineSearchPath &searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &visitedFunctions, int &running_instcount, llvm::StringSet<> &importVars)
 
static void function_references (llvm::Function &F, int &running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &referencedFunctions)
 
static void add_module_to_inline_search_path (InlineSearchPath &path, llvm::StringRef modpath)
 
static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid (const InlineSearchPath &path, llvm::GlobalValue::GUID guid)
 
void llvm_inline (LLVMModuleRef M)
 

Variables

const float inline_cost_decay_factor = 0.5
 
const int inline_initial_cost = 150
 
llvm::ManagedStatic< ModuleCachemodule_cache
 
llvm::ManagedStatic< SummaryCachesummary_cache
 

Macro Definition Documentation

◆ hasFnAttr

#define hasFnAttr   hasFnAttribute

◆ ilog

#define ilog (   ...)    (void) 0

Definition at line 152 of file llvmjit_inline.cpp.

◆ IRMOVE_PARAMS

#define IRMOVE_PARAMS

Typedef Documentation

◆ FunctionInlineState

◆ FunctionInlineStates

typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates

Definition at line 95 of file llvmjit_inline.cpp.

◆ ImportMapTy

typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy

Definition at line 101 of file llvmjit_inline.cpp.

◆ InlineSearchPath

typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath

Definition at line 72 of file llvmjit_inline.cpp.

◆ InlineWorkList

typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList

Definition at line 82 of file llvmjit_inline.cpp.

◆ InlineWorkListItem

◆ ModuleCache

typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache

Definition at line 111 of file llvmjit_inline.cpp.

◆ SummaryCache

typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache

Definition at line 113 of file llvmjit_inline.cpp.

Function Documentation

◆ add_module_to_inline_search_path()

static void add_module_to_inline_search_path ( InlineSearchPath path,
llvm::StringRef  modpath 
)
static

Definition at line 808 of file llvmjit_inline.cpp.

809 {
810  /* only extension in libdir are candidates for inlining for now */
811  if (!modpath.startswith("$libdir/"))
812  return;
813 
814  /* if there's no match, attempt to load */
815  auto it = summary_cache->find(modpath);
816  if (it == summary_cache->end())
817  {
818  std::string path(modpath);
819  path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
820  path += ".index.bc";
821  (*summary_cache)[modpath] = llvm_load_summary(path);
822  it = summary_cache->find(modpath);
823  }
824 
825  Assert(it != summary_cache->end());
826 
827  /* if the entry isn't NULL, it's validly loaded */
828  if (it->second)
829  searchpath.push_back(it->second.get());
830 }
char pkglib_path[MAXPGPATH]
Definition: globals.c:77
Assert(fmt[strlen(fmt) - 1] !='\n')
static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary(llvm::StringRef path)
llvm::ManagedStatic< SummaryCache > summary_cache
char string[11]
Definition: preproc-type.c:46

References Assert(), llvm_load_summary(), pkglib_path, and summary_cache.

Referenced by llvm_build_inline_plan().

◆ create_redirection_function()

static llvm::Function * create_redirection_function ( std::unique_ptr< llvm::Module > &  importMod,
llvm::Function *  F,
llvm::StringRef  Name 
)
static

Definition at line 868 of file llvmjit_inline.cpp.

871 {
872  typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
873 
874  llvm::LLVMContext &Context = F->getContext();
875  llvm::IRBuilder<> Builder(Context);
876  llvm::Function *AF;
877  llvm::BasicBlock *BB;
878  llvm::CallInst *fwdcall;
879 #if LLVM_VERSION_MAJOR < 14
880  llvm::Attribute inlineAttribute;
881 #endif
882 
883  AF = llvm::Function::Create(F->getFunctionType(),
884  LinkageTypes::AvailableExternallyLinkage,
885  Name, importMod.get());
886  BB = llvm::BasicBlock::Create(Context, "entry", AF);
887 
888  Builder.SetInsertPoint(BB);
889  fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
890 #if LLVM_VERSION_MAJOR < 14
891  inlineAttribute = llvm::Attribute::get(Context,
892  llvm::Attribute::AlwaysInline);
893  fwdcall->addAttribute(~0U, inlineAttribute);
894 #else
895  fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
896 #endif
897  Builder.CreateRet(fwdcall);
898 
899  return AF;
900 }
#define F(X, Y, Z)
Definition: md5.c:60
Definition: c.h:687

References F.

Referenced by llvm_execute_inline_plan().

◆ function_inlinable()

static bool function_inlinable ( llvm::Function &  F,
int  threshold,
FunctionInlineStates functionState,
InlineWorkList worklist,
InlineSearchPath searchpath,
llvm::SmallPtrSet< const llvm::Function *, 8 > &  visitedFunctions,
int &  running_instcount,
llvm::StringSet<> &  importVars 
)
static

Definition at line 569 of file llvmjit_inline.cpp.

577 {
578  int subThreshold = threshold * inline_cost_decay_factor;
579  llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
580  llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
581 
582  /* can't rely on what may be inlined */
583  if (F.isInterposable())
584  return false;
585 
586  /*
587  * Can't rely on function being present. Alternatively we could create a
588  * static version of these functions?
589  */
590  if (F.hasAvailableExternallyLinkage())
591  return false;
592 
593  ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
594 
595  if (F.materialize())
596  elog(FATAL, "failed to materialize metadata");
597 
598 #if LLVM_VERSION_MAJOR < 14
599 #define hasFnAttr hasFnAttribute
600 #endif
601 
602  if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
603  {
604  ilog(DEBUG1, "ineligibile to import %s due to noinline",
605  F.getName().data());
606  return false;
607  }
608 
609  function_references(F, running_instcount, referencedVars, referencedFunctions);
610 
611  for (llvm::GlobalVariable* rv: referencedVars)
612  {
613  if (rv->materialize())
614  elog(FATAL, "failed to materialize metadata");
615 
616  /*
617  * Don't inline functions that access thread local variables. That
618  * doesn't work on current LLVM releases (but might in future).
619  */
620  if (rv->isThreadLocal())
621  {
622  ilog(DEBUG1, "cannot inline %s due to thread-local variable %s",
623  F.getName().data(), rv->getName().data());
624  return false;
625  }
626 
627  /*
628  * Never want to inline externally visible vars, cheap enough to
629  * reference.
630  */
631  if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
632  continue;
633 
634  /*
635  * If variable is file-local, we need to inline it, to be able to
636  * inline the function itself. Can't do that if the variable can be
637  * modified, because they'd obviously get out of sync.
638  *
639  * XXX: Currently not a problem, but there'd be problems with
640  * nontrivial initializers if they were allowed for postgres.
641  */
642  if (!rv->isConstant())
643  {
644  ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
645  F.getName().data(), rv->getName().data());
646  return false;
647  }
648 
649  ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
650  rv->getName().data(), (int)rv->getLinkage());
651 
652  importVars.insert(rv->getName());
653  /* small cost attributed to each cloned global */
654  running_instcount += 5;
655  }
656 
657  visitedFunctions.insert(&F);
658 
659  /*
660  * Check referenced functions. Check whether used static ones are
661  * inlinable, and remember external ones for inlining.
662  */
663  for (llvm::Function* referencedFunction: referencedFunctions)
664  {
665  llvm::StringSet<> recImportVars;
666 
667  if (referencedFunction->materialize())
668  elog(FATAL, "failed to materialize metadata");
669 
670  if (referencedFunction->isIntrinsic())
671  continue;
672 
673  /* if already visited skip, otherwise remember */
674  if (!visitedFunctions.insert(referencedFunction).second)
675  continue;
676 
677  /*
678  * We don't inline external functions directly here, instead we put
679  * them on the worklist if appropriate and check them from
680  * llvm_build_inline_plan().
681  */
682  if (referencedFunction->hasExternalLinkage())
683  {
684  llvm::StringRef funcName = referencedFunction->getName();
685 
686  /*
687  * Don't bother checking for inlining if remaining cost budget is
688  * very small.
689  */
690  if (subThreshold < 5)
691  continue;
692 
693  auto it = functionStates.find(funcName);
694  if (it == functionStates.end())
695  {
696  FunctionInlineState inlineState;
697 
698  inlineState.costLimit = subThreshold;
699  inlineState.processed = false;
700  inlineState.inlined = false;
701  inlineState.allowReconsidering = false;
702 
703  functionStates[funcName] = inlineState;
704  worklist.push_back({funcName, searchpath});
705 
706  ilog(DEBUG1,
707  "considering extern function %s at %d for inlining",
708  funcName.data(), subThreshold);
709  }
710  else if (!it->second.inlined &&
711  (!it->second.processed || it->second.allowReconsidering) &&
712  it->second.costLimit < subThreshold)
713  {
714  /*
715  * Update inlining threshold if higher. Need to re-queue
716  * to be processed if already processed with lower
717  * threshold.
718  */
719  if (it->second.processed)
720  {
721  ilog(DEBUG1,
722  "reconsidering extern function %s at %d for inlining, increasing from %d",
723  funcName.data(), subThreshold, it->second.costLimit);
724 
725  it->second.processed = false;
726  it->second.allowReconsidering = false;
727  worklist.push_back({funcName, searchpath});
728  }
729  it->second.costLimit = subThreshold;
730  }
731  continue;
732  }
733 
734  /* can't rely on what may be inlined */
735  if (referencedFunction->isInterposable())
736  return false;
737 
738  if (!function_inlinable(*referencedFunction,
739  subThreshold,
740  functionStates,
741  worklist,
742  searchpath,
743  visitedFunctions,
744  running_instcount,
745  recImportVars))
746  {
747  ilog(DEBUG1,
748  "cannot inline %s due to required function %s not being inlinable",
749  F.getName().data(), referencedFunction->getName().data());
750  return false;
751  }
752 
753  /* import referenced function itself */
754  importVars.insert(referencedFunction->getName());
755 
756  /* import referenced function and its dependants */
757  for (auto& recImportVar : recImportVars)
758  importVars.insert(recImportVar.first());
759  }
760 
761  return true;
762 }
#define FATAL
Definition: elog.h:35
#define DEBUG1
Definition: elog.h:24
#define elog(elevel,...)
Definition: elog.h:218
const float inline_cost_decay_factor
static void function_references(llvm::Function &F, int &running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &referencedFunctions)
#define ilog(...)
static bool function_inlinable(llvm::Function &F, int threshold, FunctionInlineStates &functionState, InlineWorkList &worklist, InlineSearchPath &searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &visitedFunctions, int &running_instcount, llvm::StringSet<> &importVars)

References FunctionInlineState::allowReconsidering, FunctionInlineState::costLimit, DEBUG1, elog, F, FATAL, function_references(), ilog, inline_cost_decay_factor, FunctionInlineState::inlined, and FunctionInlineState::processed.

Referenced by llvm_build_inline_plan().

◆ function_references()

static void function_references ( llvm::Function &  F,
int &  running_instcount,
llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &  referencedVars,
llvm::SmallPtrSet< llvm::Function *, 8 > &  referencedFunctions 
)
static

Definition at line 512 of file llvmjit_inline.cpp.

516 {
517  llvm::SmallPtrSet<const llvm::User *, 32> Visited;
518 
519  for (llvm::BasicBlock &BB : F)
520  {
521  for (llvm::Instruction &I : BB)
522  {
523  if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
524  continue;
525 
526  llvm::SmallVector<llvm::User *, 8> Worklist;
527  Worklist.push_back(&I);
528 
529  running_instcount++;
530 
531  while (!Worklist.empty()) {
532  llvm::User *U = Worklist.pop_back_val();
533 
534  /* visited before */
535  if (!Visited.insert(U).second)
536  continue;
537 
538  for (auto &OI : U->operands()) {
539  llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
540  if (!Operand)
541  continue;
542  if (llvm::isa<llvm::BlockAddress>(Operand))
543  continue;
544  if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
545  referencedVars.insert(GV);
546  if (GV->hasInitializer())
547  Worklist.push_back(GV->getInitializer());
548  continue;
549  }
550  if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
551  referencedFunctions.insert(CF);
552  continue;
553  }
554  Worklist.push_back(Operand);
555  }
556  }
557  }
558  }
559 }
#define I(X, Y, Z)
Definition: md5.c:63

References F, and I.

Referenced by function_inlinable().

◆ llvm_build_inline_plan()

static std::unique_ptr< ImportMapTy > llvm_build_inline_plan ( llvm::Module *  mod)
static

Definition at line 175 of file llvmjit_inline.cpp.

176 {
177  std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
178  FunctionInlineStates functionStates;
179  InlineWorkList worklist;
180 
181  InlineSearchPath defaultSearchPath;
182 
183  /* attempt to add module to search path */
184  add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
185  /* if postgres isn't available, no point continuing */
186  if (defaultSearchPath.empty())
187  return nullptr;
188 
189  /*
190  * Start inlining with current references to external functions by putting
191  * them on the inlining worklist. If, during inlining of those, new extern
192  * functions need to be inlined, they'll also be put there, with a lower
193  * priority.
194  */
195  for (const llvm::Function &funcDecl : mod->functions())
196  {
197  InlineWorkListItem item = {};
198  FunctionInlineState inlineState = {};
199 
200  /* already has a definition */
201  if (!funcDecl.isDeclaration())
202  continue;
203 
204  /* llvm provides implementation */
205  if (funcDecl.isIntrinsic())
206  continue;
207 
208  item.symbolName = funcDecl.getName();
209  item.searchpath = defaultSearchPath;
210  worklist.push_back(item);
211  inlineState.costLimit = inline_initial_cost;
212  inlineState.processed = false;
213  inlineState.inlined = false;
214  inlineState.allowReconsidering = false;
215  functionStates[funcDecl.getName()] = inlineState;
216  }
217 
218  /*
219  * Iterate over pending worklist items, look them up in index, check
220  * whether they should be inlined.
221  */
222  while (!worklist.empty())
223  {
224  InlineWorkListItem item = worklist.pop_back_val();
225  llvm::StringRef symbolName = item.symbolName;
226  char *cmodname;
227  char *cfuncname;
228  FunctionInlineState &inlineState = functionStates[symbolName];
229  llvm::GlobalValue::GUID funcGUID;
230 
231  llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
232 
233  funcGUID = llvm::GlobalValue::getGUID(cfuncname);
234 
235  /* already processed */
236  if (inlineState.processed)
237  continue;
238 
239 
240  if (cmodname)
242 
243  /*
244  * Iterate over all known definitions of function, via the index. Then
245  * look up module(s), check if function actually is defined (there
246  * could be hash conflicts).
247  */
248  for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
249  {
250  const llvm::FunctionSummary *fs;
251  llvm::StringRef modPath = gvs->modulePath();
252  llvm::Module *defMod;
253  llvm::Function *funcDef;
254 
255  fs = llvm::cast<llvm::FunctionSummary>(gvs);
256 
257 #if LLVM_VERSION_MAJOR > 3
258  if (gvs->notEligibleToImport())
259  {
260  ilog(DEBUG1, "ineligibile to import %s due to summary",
261  symbolName.data());
262  continue;
263  }
264 #endif
265 
266  if ((int) fs->instCount() > inlineState.costLimit)
267  {
268  ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
269  symbolName.data(), fs->instCount(), inlineState.costLimit);
270  inlineState.allowReconsidering = true;
271  continue;
272  }
273 
274  defMod = load_module_cached(modPath);
275  if (defMod->materializeMetadata())
276  elog(FATAL, "failed to materialize metadata");
277 
278  funcDef = defMod->getFunction(cfuncname);
279 
280  /*
281  * This can happen e.g. in case of a hash collision of the
282  * function's name.
283  */
284  if (!funcDef)
285  continue;
286 
287  if (funcDef->materialize())
288  elog(FATAL, "failed to materialize metadata");
289 
290  Assert(!funcDef->isDeclaration());
291  Assert(funcDef->hasExternalLinkage());
292 
293  llvm::StringSet<> importVars;
294  llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
295  int running_instcount = 0;
296 
297  /*
298  * Check whether function, and objects it depends on, are
299  * inlinable.
300  */
301  if (function_inlinable(*funcDef,
302  inlineState.costLimit,
303  functionStates,
304  worklist,
305  item.searchpath,
306  visitedFunctions,
307  running_instcount,
308  importVars))
309  {
310  /*
311  * Check whether function and all its dependencies are too
312  * big. Dependencies already counted for other functions that
313  * will get inlined are not counted again. While this make
314  * things somewhat order dependent, I can't quite see a point
315  * in a different behaviour.
316  */
317  if (running_instcount > inlineState.costLimit)
318  {
319  ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
320  symbolName.data(), running_instcount, inlineState.costLimit);
321  inlineState.allowReconsidering = true;
322  continue;
323  }
324 
325  ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
326  symbolName.data(), running_instcount, fs->instCount());
327 
328  /* import referenced function itself */
329  importVars.insert(symbolName);
330 
331  {
332  llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
333  for (auto& importVar : importVars)
334  modGlobalsToInline.insert(importVar.first());
335  Assert(modGlobalsToInline.size() > 0);
336  }
337 
338  /* mark function as inlined */
339  inlineState.inlined = true;
340 
341  /*
342  * Found definition to inline, don't look for further
343  * potential definitions.
344  */
345  break;
346  }
347  else
348  {
349  ilog(DEBUG1, "had to skip inlining %s",
350  symbolName.data());
351 
352  /* It's possible there's another definition that's inlinable. */
353  }
354  }
355 
356  /*
357  * Signal that we're done with symbol, whether successful (inlined =
358  * true above) or not.
359  */
360  inlineState.processed = true;
361  }
362 
363  return globalsToInline;
364 }
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
Definition: llvmjit.c:1035
llvm::SmallVector< InlineWorkListItem, 128 > InlineWorkList
const int inline_initial_cost
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > InlineSearchPath
static llvm::Module * load_module_cached(llvm::StringRef modPath)
static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid(const InlineSearchPath &path, llvm::GlobalValue::GUID guid)
llvm::StringMap< llvm::StringSet<> > ImportMapTy
static void add_module_to_inline_search_path(InlineSearchPath &path, llvm::StringRef modpath)
llvm::StringMap< FunctionInlineState > FunctionInlineStates
llvm::StringRef symbolName
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > searchpath

References add_module_to_inline_search_path(), FunctionInlineState::allowReconsidering, Assert(), FunctionInlineState::costLimit, DEBUG1, elog, FATAL, function_inlinable(), ilog, inline_initial_cost, FunctionInlineState::inlined, llvm_split_symbol_name(), load_module_cached(), FunctionInlineState::processed, InlineWorkListItem::searchpath, summaries_for_guid(), and InlineWorkListItem::symbolName.

Referenced by llvm_inline().

◆ llvm_execute_inline_plan()

static void llvm_execute_inline_plan ( llvm::Module *  mod,
ImportMapTy globalsToInline 
)
static

Definition at line 371 of file llvmjit_inline.cpp.

372 {
373  llvm::IRMover Mover(*mod);
374 
375  for (const auto& toInline : *globalsToInline)
376  {
377  const llvm::StringRef& modPath = toInline.first();
378  const llvm::StringSet<>& modGlobalsToInline = toInline.second;
379  llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
380 
381  Assert(module_cache->count(modPath));
382  std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
383  module_cache->erase(modPath);
384 
385  if (modGlobalsToInline.empty())
386  continue;
387 
388  for (auto &glob: modGlobalsToInline)
389  {
390  llvm::StringRef SymbolName = glob.first();
391  char *modname;
392  char *funcname;
393 
394  llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
395 
396  llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
397 
398  if (!valueToImport)
399  elog(FATAL, "didn't refind value %s to import", SymbolName.data());
400 
401  /*
402  * For functions (global vars are only inlined if already static),
403  * mark imported variables as being clones from other
404  * functions. That a) avoids symbol conflicts b) allows the
405  * optimizer to perform inlining.
406  */
407  if (llvm::isa<llvm::Function>(valueToImport))
408  {
409  llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
410  typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
411 
412  /*
413  * Per-function info isn't necessarily stripped yet, as the
414  * module is lazy-loaded when stripped above.
415  */
416  llvm::stripDebugInfo(*F);
417 
418  /*
419  * If the to-be-imported function is one referenced including
420  * its module name, create a tiny inline function that just
421  * forwards the call. One might think a GlobalAlias would do
422  * the trick, but a) IRMover doesn't override a declaration
423  * with an alias pointing to a definition (instead renaming
424  * it), b) Aliases can't be AvailableExternally.
425  */
426  if (modname)
427  {
428  llvm::Function *AF;
429 
430  AF = create_redirection_function(importMod, F, SymbolName);
431 
432  GlobalsToImport.insert(AF);
433  llvm::stripDebugInfo(*AF);
434  }
435 
436  if (valueToImport->hasExternalLinkage())
437  {
438  valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
439  }
440  }
441 
442  GlobalsToImport.insert(valueToImport);
443  ilog(DEBUG1, "performing import of %s %s",
444  modPath.data(), SymbolName.data());
445 
446  }
447 
448 #if LLVM_VERSION_MAJOR > 4
449 #define IRMOVE_PARAMS , /*IsPerformingImport=*/false
450 #elif LLVM_VERSION_MAJOR > 3
451 #define IRMOVE_PARAMS , /*LinkModuleInlineAsm=*/false, /*IsPerformingImport=*/false
452 #else
453 #define IRMOVE_PARAMS
454 #endif
455  if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
456  [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}
457  IRMOVE_PARAMS))
458  elog(FATAL, "function import failed with linker error");
459  }
460 }
static llvm::Function * create_redirection_function(std::unique_ptr< llvm::Module > &importMod, llvm::Function *F, llvm::StringRef Name)
llvm::ManagedStatic< ModuleCache > module_cache
#define IRMOVE_PARAMS

References Assert(), create_redirection_function(), DEBUG1, elog, F, FATAL, ilog, IRMOVE_PARAMS, llvm_split_symbol_name(), and module_cache.

Referenced by llvm_inline().

◆ llvm_inline()

void llvm_inline ( LLVMModuleRef  M)

Definition at line 160 of file llvmjit_inline.cpp.

161 {
162  llvm::Module *mod = llvm::unwrap(M);
163 
164  std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(mod);
165  if (!globalsToInline)
166  return;
167  llvm_execute_inline_plan(mod, globalsToInline.get());
168 }
static std::unique_ptr< ImportMapTy > llvm_build_inline_plan(llvm::Module *mod)
static void llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)

References llvm_build_inline_plan(), and llvm_execute_inline_plan().

Referenced by llvm_compile_module().

◆ llvm_load_summary()

static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary ( llvm::StringRef  path)
static

Definition at line 769 of file llvmjit_inline.cpp.

770 {
771  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
772  llvm::MemoryBuffer::getFile(path);
773 
774  if (std::error_code EC = MBOrErr.getError())
775  {
776  ilog(DEBUG1, "failed to open %s: %s", path.data(),
777  EC.message().c_str());
778  }
779  else
780  {
781  llvm::MemoryBufferRef ref(*MBOrErr.get().get());
782 
783 #if LLVM_VERSION_MAJOR > 3
784  llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
785  llvm::getModuleSummaryIndex(ref);
786  if (IndexOrErr)
787  return std::move(IndexOrErr.get());
788  elog(FATAL, "failed to load summary \"%s\": %s",
789  path.data(),
790  toString(IndexOrErr.takeError()).c_str());
791 #else
792  llvm::ErrorOr<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
793  llvm::getModuleSummaryIndex(ref, [](const llvm::DiagnosticInfo &) {});
794  if (IndexOrErr)
795  return std::move(IndexOrErr.get());
796  elog(FATAL, "failed to load summary \"%s\": %s",
797  path.data(),
798  IndexOrErr.getError().message().c_str());
799 #endif
800  }
801  return nullptr;
802 }

References DEBUG1, elog, FATAL, and ilog.

Referenced by add_module_to_inline_search_path().

◆ load_module()

static std::unique_ptr< llvm::Module > load_module ( llvm::StringRef  Identifier)
static

Definition at line 482 of file llvmjit_inline.cpp.

483 {
484  LLVMMemoryBufferRef buf;
485  LLVMModuleRef mod;
486  char path[MAXPGPATH];
487  char *msg;
488 
489  snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
490 
491  if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
492  elog(FATAL, "failed to open bitcode file \"%s\": %s",
493  path, msg);
494  if (LLVMGetBitcodeModuleInContext2(LLVMGetGlobalContext(), buf, &mod))
495  elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
496 
497  /*
498  * Currently there's no use in more detailed debug info for JITed
499  * code. Until that changes, not much point in wasting memory and cycles
500  * on processing debuginfo.
501  */
502  llvm::StripDebugInfo(*llvm::unwrap(mod));
503 
504  return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
505 }
#define MAXPGPATH
static char * buf
Definition: pg_test_fsync.c:67
#define snprintf
Definition: port.h:225

References buf, elog, FATAL, MAXPGPATH, pkglib_path, and snprintf.

Referenced by load_module_cached().

◆ load_module_cached()

static llvm::Module * load_module_cached ( llvm::StringRef  modPath)
static

Definition at line 469 of file llvmjit_inline.cpp.

470 {
471  auto it = module_cache->find(modPath);
472  if (it == module_cache->end())
473  {
474  it = module_cache->insert(
475  std::make_pair(modPath, load_module(modPath))).first;
476  }
477 
478  return it->second.get();
479 }
static std::unique_ptr< llvm::Module > load_module(llvm::StringRef Identifier)

References load_module(), and module_cache.

Referenced by llvm_build_inline_plan().

◆ summaries_for_guid()

static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid ( const InlineSearchPath path,
llvm::GlobalValue::GUID  guid 
)
static

Definition at line 837 of file llvmjit_inline.cpp.

838 {
839  llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
840 
841  for (auto index : path)
842  {
843 #if LLVM_VERSION_MAJOR > 4
844  llvm::ValueInfo funcVI = index->getValueInfo(guid);
845 
846  /* if index doesn't know function, we don't have a body, continue */
847  if (funcVI)
848  for (auto &gv : funcVI.getSummaryList())
849  matches.push_back(gv.get());
850 #else
851  const llvm::const_gvsummary_iterator &I =
852  index->findGlobalValueSummaryList(guid);
853  if (I != index->end())
854  {
855  for (auto &gv : I->second)
856  matches.push_back(gv.get());
857  }
858 #endif
859  }
860 
861  return matches;
862 }
Definition: type.h:90

References I.

Referenced by llvm_build_inline_plan().

Variable Documentation

◆ inline_cost_decay_factor

const float inline_cost_decay_factor = 0.5

Definition at line 104 of file llvmjit_inline.cpp.

Referenced by function_inlinable().

◆ inline_initial_cost

const int inline_initial_cost = 150

Definition at line 105 of file llvmjit_inline.cpp.

Referenced by llvm_build_inline_plan().

◆ module_cache

llvm::ManagedStatic<ModuleCache> module_cache

Definition at line 112 of file llvmjit_inline.cpp.

Referenced by llvm_execute_inline_plan(), and load_module_cached().

◆ summary_cache

llvm::ManagedStatic<SummaryCache> summary_cache

Definition at line 114 of file llvmjit_inline.cpp.

Referenced by add_module_to_inline_search_path().