PostgreSQL Source Code  git master
llvmjit_inline.cpp File Reference
#include "postgres.h"
#include "jit/llvmjit.h"
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "common/string.h"
#include "miscadmin.h"
#include "storage/fd.h"
#include <llvm-c/Core.h>
#include <llvm-c/BitReader.h>
#include <llvm/ADT/SetVector.h>
#include <llvm/ADT/StringSet.h>
#include <llvm/ADT/StringMap.h>
#include <llvm/Analysis/ModuleSummaryAnalysis.h>
#include <llvm/Bitcode/ReaderWriter.h>
#include <llvm/Support/Error.h>
#include <llvm/IR/Attributes.h>
#include <llvm/IR/DebugInfo.h>
#include <llvm/IR/IntrinsicInst.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/ModuleSummaryIndex.h>
#include <llvm/Linker/IRMover.h>
#include <llvm/Support/ManagedStatic.h>
#include <llvm/Support/MemoryBuffer.h>
Include dependency graph for llvmjit_inline.cpp:

Go to the source code of this file.

Data Structures

struct  InlineWorkListItem
 
struct  FunctionInlineState
 

Macros

#define ilog(...)   (void) 0
 
#define IRMOVE_PARAMS
 
#define hasFnAttr   hasFnAttribute
 

Typedefs

typedef llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > InlineSearchPath
 
typedef struct InlineWorkListItem InlineWorkListItem
 
typedef llvm::SmallVector< InlineWorkListItem, 128 > InlineWorkList
 
typedef struct FunctionInlineState FunctionInlineState
 
typedef llvm::StringMap< FunctionInlineStateFunctionInlineStates
 
typedef llvm::StringMap< llvm::StringSet<> > ImportMapTy
 
typedef llvm::StringMap< std::unique_ptr< llvm::Module > > ModuleCache
 
typedef llvm::StringMap< std::unique_ptr< llvm::ModuleSummaryIndex > > SummaryCache
 

Functions

static std::unique_ptr< ImportMapTyllvm_build_inline_plan (LLVMContextRef lc, llvm::Module *mod)
 
static void llvm_execute_inline_plan (llvm::Module *mod, ImportMapTy *globalsToInline)
 
static llvm::Module * load_module_cached (LLVMContextRef c, llvm::StringRef modPath)
 
static std::unique_ptr< llvm::Module > load_module (LLVMContextRef c, llvm::StringRef Identifier)
 
static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary (llvm::StringRef path)
 
static llvm::Function * create_redirection_function (std::unique_ptr< llvm::Module > &importMod, llvm::Function *F, llvm::StringRef Name)
 
static bool function_inlinable (llvm::Function &F, int threshold, FunctionInlineStates &functionState, InlineWorkList &worklist, InlineSearchPath &searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &visitedFunctions, int &running_instcount, llvm::StringSet<> &importVars)
 
static void function_references (llvm::Function &F, int &running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &referencedFunctions)
 
static void add_module_to_inline_search_path (InlineSearchPath &path, llvm::StringRef modpath)
 
static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid (const InlineSearchPath &path, llvm::GlobalValue::GUID guid)
 
void llvm_inline_reset_caches (void)
 
void llvm_inline (LLVMModuleRef M)
 

Variables

const float inline_cost_decay_factor = 0.5
 
const int inline_initial_cost = 150
 
llvm::ManagedStatic< ModuleCachemodule_cache
 
llvm::ManagedStatic< SummaryCachesummary_cache
 

Macro Definition Documentation

◆ hasFnAttr

#define hasFnAttr   hasFnAttribute

◆ ilog

#define ilog (   ...)    (void) 0

Definition at line 152 of file llvmjit_inline.cpp.

◆ IRMOVE_PARAMS

#define IRMOVE_PARAMS

Typedef Documentation

◆ FunctionInlineState

◆ FunctionInlineStates

typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates

Definition at line 95 of file llvmjit_inline.cpp.

◆ ImportMapTy

typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy

Definition at line 101 of file llvmjit_inline.cpp.

◆ InlineSearchPath

typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath

Definition at line 72 of file llvmjit_inline.cpp.

◆ InlineWorkList

typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList

Definition at line 82 of file llvmjit_inline.cpp.

◆ InlineWorkListItem

◆ ModuleCache

typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache

Definition at line 111 of file llvmjit_inline.cpp.

◆ SummaryCache

typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache

Definition at line 113 of file llvmjit_inline.cpp.

Function Documentation

◆ add_module_to_inline_search_path()

static void add_module_to_inline_search_path ( InlineSearchPath path,
llvm::StringRef  modpath 
)
static

Definition at line 821 of file llvmjit_inline.cpp.

822 {
823  /* only extension in libdir are candidates for inlining for now */
824  if (!modpath.startswith("$libdir/"))
825  return;
826 
827  /* if there's no match, attempt to load */
828  auto it = summary_cache->find(modpath);
829  if (it == summary_cache->end())
830  {
831  std::string path(modpath);
832  path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
833  path += ".index.bc";
834  (*summary_cache)[modpath] = llvm_load_summary(path);
835  it = summary_cache->find(modpath);
836  }
837 
838  Assert(it != summary_cache->end());
839 
840  /* if the entry isn't NULL, it's validly loaded */
841  if (it->second)
842  searchpath.push_back(it->second.get());
843 }
char pkglib_path[MAXPGPATH]
Definition: globals.c:77
Assert(fmt[strlen(fmt) - 1] !='\n')
static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary(llvm::StringRef path)
llvm::ManagedStatic< SummaryCache > summary_cache
char string[11]
Definition: preproc-type.c:52

References Assert(), llvm_load_summary(), pkglib_path, and summary_cache.

Referenced by llvm_build_inline_plan().

◆ create_redirection_function()

static llvm::Function * create_redirection_function ( std::unique_ptr< llvm::Module > &  importMod,
llvm::Function *  F,
llvm::StringRef  Name 
)
static

Definition at line 881 of file llvmjit_inline.cpp.

884 {
885  typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
886 
887  llvm::LLVMContext &Context = F->getContext();
888  llvm::IRBuilder<> Builder(Context);
889  llvm::Function *AF;
890  llvm::BasicBlock *BB;
891  llvm::CallInst *fwdcall;
892 #if LLVM_VERSION_MAJOR < 14
893  llvm::Attribute inlineAttribute;
894 #endif
895 
896  AF = llvm::Function::Create(F->getFunctionType(),
897  LinkageTypes::AvailableExternallyLinkage,
898  Name, importMod.get());
899  BB = llvm::BasicBlock::Create(Context, "entry", AF);
900 
901  Builder.SetInsertPoint(BB);
902  fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
903 #if LLVM_VERSION_MAJOR < 14
904  inlineAttribute = llvm::Attribute::get(Context,
905  llvm::Attribute::AlwaysInline);
906  fwdcall->addAttribute(~0U, inlineAttribute);
907 #else
908  fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
909 #endif
910  Builder.CreateRet(fwdcall);
911 
912  return AF;
913 }
#define F(X, Y, Z)
Definition: md5.c:60
Definition: c.h:730

References F.

Referenced by llvm_execute_inline_plan().

◆ function_inlinable()

static bool function_inlinable ( llvm::Function &  F,
int  threshold,
FunctionInlineStates functionState,
InlineWorkList worklist,
InlineSearchPath searchpath,
llvm::SmallPtrSet< const llvm::Function *, 8 > &  visitedFunctions,
int &  running_instcount,
llvm::StringSet<> &  importVars 
)
static

Definition at line 582 of file llvmjit_inline.cpp.

590 {
591  int subThreshold = threshold * inline_cost_decay_factor;
592  llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
593  llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
594 
595  /* can't rely on what may be inlined */
596  if (F.isInterposable())
597  return false;
598 
599  /*
600  * Can't rely on function being present. Alternatively we could create a
601  * static version of these functions?
602  */
603  if (F.hasAvailableExternallyLinkage())
604  return false;
605 
606  ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
607 
608  if (F.materialize())
609  elog(FATAL, "failed to materialize metadata");
610 
611 #if LLVM_VERSION_MAJOR < 14
612 #define hasFnAttr hasFnAttribute
613 #endif
614 
615  if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
616  {
617  ilog(DEBUG1, "ineligibile to import %s due to noinline",
618  F.getName().data());
619  return false;
620  }
621 
622  function_references(F, running_instcount, referencedVars, referencedFunctions);
623 
624  for (llvm::GlobalVariable* rv: referencedVars)
625  {
626  if (rv->materialize())
627  elog(FATAL, "failed to materialize metadata");
628 
629  /*
630  * Don't inline functions that access thread local variables. That
631  * doesn't work on current LLVM releases (but might in future).
632  */
633  if (rv->isThreadLocal())
634  {
635  ilog(DEBUG1, "cannot inline %s due to thread-local variable %s",
636  F.getName().data(), rv->getName().data());
637  return false;
638  }
639 
640  /*
641  * Never want to inline externally visible vars, cheap enough to
642  * reference.
643  */
644  if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
645  continue;
646 
647  /*
648  * If variable is file-local, we need to inline it, to be able to
649  * inline the function itself. Can't do that if the variable can be
650  * modified, because they'd obviously get out of sync.
651  *
652  * XXX: Currently not a problem, but there'd be problems with
653  * nontrivial initializers if they were allowed for postgres.
654  */
655  if (!rv->isConstant())
656  {
657  ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
658  F.getName().data(), rv->getName().data());
659  return false;
660  }
661 
662  ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
663  rv->getName().data(), (int)rv->getLinkage());
664 
665  importVars.insert(rv->getName());
666  /* small cost attributed to each cloned global */
667  running_instcount += 5;
668  }
669 
670  visitedFunctions.insert(&F);
671 
672  /*
673  * Check referenced functions. Check whether used static ones are
674  * inlinable, and remember external ones for inlining.
675  */
676  for (llvm::Function* referencedFunction: referencedFunctions)
677  {
678  llvm::StringSet<> recImportVars;
679 
680  if (referencedFunction->materialize())
681  elog(FATAL, "failed to materialize metadata");
682 
683  if (referencedFunction->isIntrinsic())
684  continue;
685 
686  /* if already visited skip, otherwise remember */
687  if (!visitedFunctions.insert(referencedFunction).second)
688  continue;
689 
690  /*
691  * We don't inline external functions directly here, instead we put
692  * them on the worklist if appropriate and check them from
693  * llvm_build_inline_plan().
694  */
695  if (referencedFunction->hasExternalLinkage())
696  {
697  llvm::StringRef funcName = referencedFunction->getName();
698 
699  /*
700  * Don't bother checking for inlining if remaining cost budget is
701  * very small.
702  */
703  if (subThreshold < 5)
704  continue;
705 
706  auto it = functionStates.find(funcName);
707  if (it == functionStates.end())
708  {
709  FunctionInlineState inlineState;
710 
711  inlineState.costLimit = subThreshold;
712  inlineState.processed = false;
713  inlineState.inlined = false;
714  inlineState.allowReconsidering = false;
715 
716  functionStates[funcName] = inlineState;
717  worklist.push_back({funcName, searchpath});
718 
719  ilog(DEBUG1,
720  "considering extern function %s at %d for inlining",
721  funcName.data(), subThreshold);
722  }
723  else if (!it->second.inlined &&
724  (!it->second.processed || it->second.allowReconsidering) &&
725  it->second.costLimit < subThreshold)
726  {
727  /*
728  * Update inlining threshold if higher. Need to re-queue
729  * to be processed if already processed with lower
730  * threshold.
731  */
732  if (it->second.processed)
733  {
734  ilog(DEBUG1,
735  "reconsidering extern function %s at %d for inlining, increasing from %d",
736  funcName.data(), subThreshold, it->second.costLimit);
737 
738  it->second.processed = false;
739  it->second.allowReconsidering = false;
740  worklist.push_back({funcName, searchpath});
741  }
742  it->second.costLimit = subThreshold;
743  }
744  continue;
745  }
746 
747  /* can't rely on what may be inlined */
748  if (referencedFunction->isInterposable())
749  return false;
750 
751  if (!function_inlinable(*referencedFunction,
752  subThreshold,
753  functionStates,
754  worklist,
755  searchpath,
756  visitedFunctions,
757  running_instcount,
758  recImportVars))
759  {
760  ilog(DEBUG1,
761  "cannot inline %s due to required function %s not being inlinable",
762  F.getName().data(), referencedFunction->getName().data());
763  return false;
764  }
765 
766  /* import referenced function itself */
767  importVars.insert(referencedFunction->getName());
768 
769  /* import referenced function and its dependents */
770  for (auto& recImportVar : recImportVars)
771  importVars.insert(recImportVar.first());
772  }
773 
774  return true;
775 }
#define FATAL
Definition: elog.h:41
#define DEBUG1
Definition: elog.h:30
const float inline_cost_decay_factor
static void function_references(llvm::Function &F, int &running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &referencedFunctions)
#define ilog(...)
static bool function_inlinable(llvm::Function &F, int threshold, FunctionInlineStates &functionState, InlineWorkList &worklist, InlineSearchPath &searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &visitedFunctions, int &running_instcount, llvm::StringSet<> &importVars)

References FunctionInlineState::allowReconsidering, FunctionInlineState::costLimit, DEBUG1, elog(), F, FATAL, function_references(), ilog, inline_cost_decay_factor, FunctionInlineState::inlined, and FunctionInlineState::processed.

Referenced by llvm_build_inline_plan().

◆ function_references()

static void function_references ( llvm::Function &  F,
int &  running_instcount,
llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &  referencedVars,
llvm::SmallPtrSet< llvm::Function *, 8 > &  referencedFunctions 
)
static

Definition at line 525 of file llvmjit_inline.cpp.

529 {
530  llvm::SmallPtrSet<const llvm::User *, 32> Visited;
531 
532  for (llvm::BasicBlock &BB : F)
533  {
534  for (llvm::Instruction &I : BB)
535  {
536  if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
537  continue;
538 
539  llvm::SmallVector<llvm::User *, 8> Worklist;
540  Worklist.push_back(&I);
541 
542  running_instcount++;
543 
544  while (!Worklist.empty()) {
545  llvm::User *U = Worklist.pop_back_val();
546 
547  /* visited before */
548  if (!Visited.insert(U).second)
549  continue;
550 
551  for (auto &OI : U->operands()) {
552  llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
553  if (!Operand)
554  continue;
555  if (llvm::isa<llvm::BlockAddress>(Operand))
556  continue;
557  if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
558  referencedVars.insert(GV);
559  if (GV->hasInitializer())
560  Worklist.push_back(GV->getInitializer());
561  continue;
562  }
563  if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
564  referencedFunctions.insert(CF);
565  continue;
566  }
567  Worklist.push_back(Operand);
568  }
569  }
570  }
571  }
572 }
#define I(X, Y, Z)
Definition: md5.c:63

References F, and I.

Referenced by function_inlinable().

◆ llvm_build_inline_plan()

static std::unique_ptr< ImportMapTy > llvm_build_inline_plan ( LLVMContextRef  lc,
llvm::Module *  mod 
)
static

Definition at line 188 of file llvmjit_inline.cpp.

189 {
190  std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
191  FunctionInlineStates functionStates;
192  InlineWorkList worklist;
193 
194  InlineSearchPath defaultSearchPath;
195 
196  /* attempt to add module to search path */
197  add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
198  /* if postgres isn't available, no point continuing */
199  if (defaultSearchPath.empty())
200  return nullptr;
201 
202  /*
203  * Start inlining with current references to external functions by putting
204  * them on the inlining worklist. If, during inlining of those, new extern
205  * functions need to be inlined, they'll also be put there, with a lower
206  * priority.
207  */
208  for (const llvm::Function &funcDecl : mod->functions())
209  {
210  InlineWorkListItem item = {};
211  FunctionInlineState inlineState = {};
212 
213  /* already has a definition */
214  if (!funcDecl.isDeclaration())
215  continue;
216 
217  /* llvm provides implementation */
218  if (funcDecl.isIntrinsic())
219  continue;
220 
221  item.symbolName = funcDecl.getName();
222  item.searchpath = defaultSearchPath;
223  worklist.push_back(item);
224  inlineState.costLimit = inline_initial_cost;
225  inlineState.processed = false;
226  inlineState.inlined = false;
227  inlineState.allowReconsidering = false;
228  functionStates[funcDecl.getName()] = inlineState;
229  }
230 
231  /*
232  * Iterate over pending worklist items, look them up in index, check
233  * whether they should be inlined.
234  */
235  while (!worklist.empty())
236  {
237  InlineWorkListItem item = worklist.pop_back_val();
238  llvm::StringRef symbolName = item.symbolName;
239  char *cmodname;
240  char *cfuncname;
241  FunctionInlineState &inlineState = functionStates[symbolName];
242  llvm::GlobalValue::GUID funcGUID;
243 
244  llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
245 
246  funcGUID = llvm::GlobalValue::getGUID(cfuncname);
247 
248  /* already processed */
249  if (inlineState.processed)
250  continue;
251 
252 
253  if (cmodname)
255 
256  /*
257  * Iterate over all known definitions of function, via the index. Then
258  * look up module(s), check if function actually is defined (there
259  * could be hash conflicts).
260  */
261  for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
262  {
263  const llvm::FunctionSummary *fs;
264  llvm::StringRef modPath = gvs->modulePath();
265  llvm::Module *defMod;
266  llvm::Function *funcDef;
267 
268  fs = llvm::cast<llvm::FunctionSummary>(gvs);
269 
270 #if LLVM_VERSION_MAJOR > 3
271  if (gvs->notEligibleToImport())
272  {
273  ilog(DEBUG1, "ineligibile to import %s due to summary",
274  symbolName.data());
275  continue;
276  }
277 #endif
278 
279  if ((int) fs->instCount() > inlineState.costLimit)
280  {
281  ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
282  symbolName.data(), fs->instCount(), inlineState.costLimit);
283  inlineState.allowReconsidering = true;
284  continue;
285  }
286 
287  defMod = load_module_cached(lc, modPath);
288  if (defMod->materializeMetadata())
289  elog(FATAL, "failed to materialize metadata");
290 
291  funcDef = defMod->getFunction(cfuncname);
292 
293  /*
294  * This can happen e.g. in case of a hash collision of the
295  * function's name.
296  */
297  if (!funcDef)
298  continue;
299 
300  if (funcDef->materialize())
301  elog(FATAL, "failed to materialize metadata");
302 
303  Assert(!funcDef->isDeclaration());
304  Assert(funcDef->hasExternalLinkage());
305 
306  llvm::StringSet<> importVars;
307  llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
308  int running_instcount = 0;
309 
310  /*
311  * Check whether function, and objects it depends on, are
312  * inlinable.
313  */
314  if (function_inlinable(*funcDef,
315  inlineState.costLimit,
316  functionStates,
317  worklist,
318  item.searchpath,
319  visitedFunctions,
320  running_instcount,
321  importVars))
322  {
323  /*
324  * Check whether function and all its dependencies are too
325  * big. Dependencies already counted for other functions that
326  * will get inlined are not counted again. While this make
327  * things somewhat order dependent, I can't quite see a point
328  * in a different behaviour.
329  */
330  if (running_instcount > inlineState.costLimit)
331  {
332  ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
333  symbolName.data(), running_instcount, inlineState.costLimit);
334  inlineState.allowReconsidering = true;
335  continue;
336  }
337 
338  ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
339  symbolName.data(), running_instcount, fs->instCount());
340 
341  /* import referenced function itself */
342  importVars.insert(symbolName);
343 
344  {
345  llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
346  for (auto& importVar : importVars)
347  modGlobalsToInline.insert(importVar.first());
348  Assert(modGlobalsToInline.size() > 0);
349  }
350 
351  /* mark function as inlined */
352  inlineState.inlined = true;
353 
354  /*
355  * Found definition to inline, don't look for further
356  * potential definitions.
357  */
358  break;
359  }
360  else
361  {
362  ilog(DEBUG1, "had to skip inlining %s",
363  symbolName.data());
364 
365  /* It's possible there's another definition that's inlinable. */
366  }
367  }
368 
369  /*
370  * Signal that we're done with symbol, whether successful (inlined =
371  * true above) or not.
372  */
373  inlineState.processed = true;
374  }
375 
376  return globalsToInline;
377 }
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
Definition: llvmjit.c:1200
llvm::SmallVector< InlineWorkListItem, 128 > InlineWorkList
const int inline_initial_cost
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > InlineSearchPath
static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid(const InlineSearchPath &path, llvm::GlobalValue::GUID guid)
llvm::StringMap< llvm::StringSet<> > ImportMapTy
static void add_module_to_inline_search_path(InlineSearchPath &path, llvm::StringRef modpath)
static llvm::Module * load_module_cached(LLVMContextRef c, llvm::StringRef modPath)
llvm::StringMap< FunctionInlineState > FunctionInlineStates
llvm::StringRef symbolName
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > searchpath

References add_module_to_inline_search_path(), FunctionInlineState::allowReconsidering, Assert(), FunctionInlineState::costLimit, DEBUG1, elog(), FATAL, function_inlinable(), ilog, inline_initial_cost, FunctionInlineState::inlined, llvm_split_symbol_name(), load_module_cached(), FunctionInlineState::processed, InlineWorkListItem::searchpath, summaries_for_guid(), and InlineWorkListItem::symbolName.

Referenced by llvm_inline().

◆ llvm_execute_inline_plan()

static void llvm_execute_inline_plan ( llvm::Module *  mod,
ImportMapTy globalsToInline 
)
static

Definition at line 384 of file llvmjit_inline.cpp.

385 {
386  llvm::IRMover Mover(*mod);
387 
388  for (const auto& toInline : *globalsToInline)
389  {
390  const llvm::StringRef& modPath = toInline.first();
391  const llvm::StringSet<>& modGlobalsToInline = toInline.second;
392  llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
393 
394  Assert(module_cache->count(modPath));
395  std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
396  module_cache->erase(modPath);
397 
398  if (modGlobalsToInline.empty())
399  continue;
400 
401  for (auto &glob: modGlobalsToInline)
402  {
403  llvm::StringRef SymbolName = glob.first();
404  char *modname;
405  char *funcname;
406 
407  llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
408 
409  llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
410 
411  if (!valueToImport)
412  elog(FATAL, "didn't refind value %s to import", SymbolName.data());
413 
414  /*
415  * For functions (global vars are only inlined if already static),
416  * mark imported variables as being clones from other
417  * functions. That a) avoids symbol conflicts b) allows the
418  * optimizer to perform inlining.
419  */
420  if (llvm::isa<llvm::Function>(valueToImport))
421  {
422  llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
423  typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
424 
425  /*
426  * Per-function info isn't necessarily stripped yet, as the
427  * module is lazy-loaded when stripped above.
428  */
429  llvm::stripDebugInfo(*F);
430 
431  /*
432  * If the to-be-imported function is one referenced including
433  * its module name, create a tiny inline function that just
434  * forwards the call. One might think a GlobalAlias would do
435  * the trick, but a) IRMover doesn't override a declaration
436  * with an alias pointing to a definition (instead renaming
437  * it), b) Aliases can't be AvailableExternally.
438  */
439  if (modname)
440  {
441  llvm::Function *AF;
442 
443  AF = create_redirection_function(importMod, F, SymbolName);
444 
445  GlobalsToImport.insert(AF);
446  llvm::stripDebugInfo(*AF);
447  }
448 
449  if (valueToImport->hasExternalLinkage())
450  {
451  valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
452  }
453  }
454 
455  GlobalsToImport.insert(valueToImport);
456  ilog(DEBUG1, "performing import of %s %s",
457  modPath.data(), SymbolName.data());
458 
459  }
460 
461 #if LLVM_VERSION_MAJOR > 4
462 #define IRMOVE_PARAMS , /*IsPerformingImport=*/false
463 #elif LLVM_VERSION_MAJOR > 3
464 #define IRMOVE_PARAMS , /*LinkModuleInlineAsm=*/false, /*IsPerformingImport=*/false
465 #else
466 #define IRMOVE_PARAMS
467 #endif
468  if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
469  [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}
470  IRMOVE_PARAMS))
471  elog(FATAL, "function import failed with linker error");
472  }
473 }
#define funcname
Definition: indent_codes.h:69
static llvm::Function * create_redirection_function(std::unique_ptr< llvm::Module > &importMod, llvm::Function *F, llvm::StringRef Name)
llvm::ManagedStatic< ModuleCache > module_cache
#define IRMOVE_PARAMS

References Assert(), create_redirection_function(), DEBUG1, elog(), F, FATAL, funcname, ilog, IRMOVE_PARAMS, llvm_split_symbol_name(), and module_cache.

Referenced by llvm_inline().

◆ llvm_inline()

void llvm_inline ( LLVMModuleRef  M)

Definition at line 172 of file llvmjit_inline.cpp.

173 {
174  LLVMContextRef lc = LLVMGetModuleContext(M);
175  llvm::Module *mod = llvm::unwrap(M);
176 
177  std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(lc, mod);
178  if (!globalsToInline)
179  return;
180  llvm_execute_inline_plan(mod, globalsToInline.get());
181 }
static std::unique_ptr< ImportMapTy > llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod)
static void llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)

References llvm_build_inline_plan(), and llvm_execute_inline_plan().

Referenced by llvm_compile_module().

◆ llvm_inline_reset_caches()

void llvm_inline_reset_caches ( void  )

Definition at line 161 of file llvmjit_inline.cpp.

162 {
163  module_cache->clear();
164  summary_cache->clear();
165 }

References module_cache, and summary_cache.

Referenced by llvm_recreate_llvm_context().

◆ llvm_load_summary()

static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary ( llvm::StringRef  path)
static

Definition at line 782 of file llvmjit_inline.cpp.

783 {
784  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
785  llvm::MemoryBuffer::getFile(path);
786 
787  if (std::error_code EC = MBOrErr.getError())
788  {
789  ilog(DEBUG1, "failed to open %s: %s", path.data(),
790  EC.message().c_str());
791  }
792  else
793  {
794  llvm::MemoryBufferRef ref(*MBOrErr.get().get());
795 
796 #if LLVM_VERSION_MAJOR > 3
797  llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
798  llvm::getModuleSummaryIndex(ref);
799  if (IndexOrErr)
800  return std::move(IndexOrErr.get());
801  elog(FATAL, "failed to load summary \"%s\": %s",
802  path.data(),
803  toString(IndexOrErr.takeError()).c_str());
804 #else
805  llvm::ErrorOr<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
806  llvm::getModuleSummaryIndex(ref, [](const llvm::DiagnosticInfo &) {});
807  if (IndexOrErr)
808  return std::move(IndexOrErr.get());
809  elog(FATAL, "failed to load summary \"%s\": %s",
810  path.data(),
811  IndexOrErr.getError().message().c_str());
812 #endif
813  }
814  return nullptr;
815 }

References DEBUG1, elog(), FATAL, and ilog.

Referenced by add_module_to_inline_search_path().

◆ load_module()

static std::unique_ptr< llvm::Module > load_module ( LLVMContextRef  c,
llvm::StringRef  Identifier 
)
static

Definition at line 495 of file llvmjit_inline.cpp.

496 {
497  LLVMMemoryBufferRef buf;
498  LLVMModuleRef mod;
499  char path[MAXPGPATH];
500  char *msg;
501 
502  snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
503 
504  if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
505  elog(FATAL, "failed to open bitcode file \"%s\": %s",
506  path, msg);
507  if (LLVMGetBitcodeModuleInContext2(lc, buf, &mod))
508  elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
509 
510  /*
511  * Currently there's no use in more detailed debug info for JITed
512  * code. Until that changes, not much point in wasting memory and cycles
513  * on processing debuginfo.
514  */
515  llvm::StripDebugInfo(*llvm::unwrap(mod));
516 
517  return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
518 }
#define MAXPGPATH
static char * buf
Definition: pg_test_fsync.c:73
#define snprintf
Definition: port.h:238

References buf, elog(), FATAL, MAXPGPATH, pkglib_path, and snprintf.

Referenced by load_module_cached().

◆ load_module_cached()

static llvm::Module * load_module_cached ( LLVMContextRef  c,
llvm::StringRef  modPath 
)
static

Definition at line 482 of file llvmjit_inline.cpp.

483 {
484  auto it = module_cache->find(modPath);
485  if (it == module_cache->end())
486  {
487  it = module_cache->insert(
488  std::make_pair(modPath, load_module(lc, modPath))).first;
489  }
490 
491  return it->second.get();
492 }
static std::unique_ptr< llvm::Module > load_module(LLVMContextRef c, llvm::StringRef Identifier)

References load_module(), and module_cache.

Referenced by llvm_build_inline_plan().

◆ summaries_for_guid()

static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid ( const InlineSearchPath path,
llvm::GlobalValue::GUID  guid 
)
static

Definition at line 850 of file llvmjit_inline.cpp.

851 {
852  llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
853 
854  for (auto index : path)
855  {
856 #if LLVM_VERSION_MAJOR > 4
857  llvm::ValueInfo funcVI = index->getValueInfo(guid);
858 
859  /* if index doesn't know function, we don't have a body, continue */
860  if (funcVI)
861  for (auto &gv : funcVI.getSummaryList())
862  matches.push_back(gv.get());
863 #else
864  const llvm::const_gvsummary_iterator &I =
865  index->findGlobalValueSummaryList(guid);
866  if (I != index->end())
867  {
868  for (auto &gv : I->second)
869  matches.push_back(gv.get());
870  }
871 #endif
872  }
873 
874  return matches;
875 }
Definition: type.h:95

References I.

Referenced by llvm_build_inline_plan().

Variable Documentation

◆ inline_cost_decay_factor

const float inline_cost_decay_factor = 0.5

Definition at line 104 of file llvmjit_inline.cpp.

Referenced by function_inlinable().

◆ inline_initial_cost

const int inline_initial_cost = 150

Definition at line 105 of file llvmjit_inline.cpp.

Referenced by llvm_build_inline_plan().

◆ module_cache

llvm::ManagedStatic<ModuleCache> module_cache

◆ summary_cache

llvm::ManagedStatic<SummaryCache> summary_cache