From 76a1c1d0ba05993f3e57f935698ec0fd09b66c8c Mon Sep 17 00:00:00 2001 From: Teresa Johnson Date: Fri, 11 Mar 2016 18:52:24 +0000 Subject: [ThinLTO] Support for reference graph in per-module and combined summary. Summary: This patch adds support for including a full reference graph including call graph edges and other GV references in the summary. The reference graph edges can be used to make importing decisions without materializing any source modules, can be used in the plugin to make file staging decisions for distributed build systems, and is expected to have other uses. The call graph edges are recorded in each function summary in the bitcode via a list of tuples when no PGO data exists, or pairs when there is PGO, where the ValueId can be mapped to the function GUID via the ValueSymbolTable. In the function index in memory, the call graph edges reference the target via the CalleeGUID instead of the CalleeValueId. The reference graph edges are recorded in each summary record with a list of referenced value IDs, which can be mapped to value GUID via the ValueSymbolTable. Addtionally, a new summary record type is added to record references from global variable initializers. A number of bitcode records and data structures have been renamed to reflect the newly expanded scope of the summary beyond functions. More cleanup will follow. Reviewers: joker.eph, davidxl Subscribers: joker.eph, llvm-commits Differential Revision: http://reviews.llvm.org/D17212 llvm-svn: 263275 --- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 465 ++++++++++++++++++++++++------ 1 file changed, 374 insertions(+), 91 deletions(-) (limited to 'llvm/lib/Bitcode/Writer/BitcodeWriter.cpp') diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 4b7b0d7..84d0963 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -11,24 +11,30 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Bitcode/ReaderWriter.h" #include "ValueEnumerator.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BlockFrequencyInfoImpl.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/Bitcode/BitstreamWriter.h" #include "llvm/Bitcode/LLVMBitCodes.h" +#include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/LLVMContext.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/IR/UseListOrder.h" #include "llvm/IR/ValueSymbolTable.h" +#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" @@ -591,11 +597,7 @@ static void writeComdats(const ValueEnumerator &VE, BitstreamWriter &Stream) { /// Write a record that will eventually hold the word offset of the /// module-level VST. For now the offset is 0, which will be backpatched /// after the real VST is written. Returns the bit offset to backpatch. -static uint64_t WriteValueSymbolTableForwardDecl(const ValueSymbolTable &VST, - BitstreamWriter &Stream) { - if (VST.empty()) - return 0; - +static uint64_t WriteValueSymbolTableForwardDecl(BitstreamWriter &Stream) { // Write a placeholder value in for the offset of the real VST, // which is written after the function blocks so that it can include // the offset of each function. The placeholder offset will be @@ -844,9 +846,11 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, Vals.clear(); } - uint64_t VSTOffsetPlaceholder = - WriteValueSymbolTableForwardDecl(M->getValueSymbolTable(), Stream); - return VSTOffsetPlaceholder; + // If we have a VST, write the VSTOFFSET record placeholder and return + // its offset. + if (M->getValueSymbolTable().empty()) + return 0; + return WriteValueSymbolTableForwardDecl(Stream); } static uint64_t GetOptimizationFlags(const Value *V) { @@ -2248,8 +2252,8 @@ static void WriteValueSymbolTable( const ValueSymbolTable &VST, const ValueEnumerator &VE, BitstreamWriter &Stream, uint64_t VSTOffsetPlaceholder = 0, uint64_t BitcodeStartBit = 0, - DenseMap> *FunctionIndex = - nullptr) { + DenseMap> + *FunctionIndex = nullptr) { if (VST.empty()) { // WriteValueSymbolTableForwardDecl should have returned early as // well. Ensure this handling remains in sync by asserting that @@ -2375,33 +2379,61 @@ static void WriteValueSymbolTable( /// Emit function names and summary offsets for the combined index /// used by ThinLTO. -static void WriteCombinedValueSymbolTable(const FunctionInfoIndex &Index, - BitstreamWriter &Stream) { +static void +WriteCombinedValueSymbolTable(const FunctionInfoIndex &Index, + BitstreamWriter &Stream, + std::map &GUIDToValueIdMap, + uint64_t VSTOffsetPlaceholder) { + assert(VSTOffsetPlaceholder > 0 && "Expected non-zero VSTOffsetPlaceholder"); + // Get the offset of the VST we are writing, and backpatch it into + // the VST forward declaration record. + uint64_t VSTOffset = Stream.GetCurrentBitNo(); + assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); + Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32); + Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcsumoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcguid - unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv); + Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_GVDEFENTRY)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // sumoffset + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // guid + unsigned DefEntryAbbrev = Stream.EmitAbbrev(Abbv); + + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid + unsigned EntryAbbrev = Stream.EmitAbbrev(Abbv); SmallVector NameVals; for (const auto &FII : Index) { + uint64_t FuncGUID = FII.first; + const auto &VMI = GUIDToValueIdMap.find(FuncGUID); + assert(VMI != GUIDToValueIdMap.end()); + for (const auto &FI : FII.second) { + // VST_CODE_COMBINED_GVDEFENTRY: [valueid, sumoffset, guid] + NameVals.push_back(VMI->second); NameVals.push_back(FI->bitcodeIndex()); - - uint64_t FuncGUID = FII.first; - - // VST_CODE_COMBINED_FNENTRY: [funcsumoffset, funcguid] - unsigned AbbrevToUse = FnEntryAbbrev; - NameVals.push_back(FuncGUID); // Emit the finished record. - Stream.EmitRecord(bitc::VST_CODE_COMBINED_FNENTRY, NameVals, AbbrevToUse); + Stream.EmitRecord(bitc::VST_CODE_COMBINED_GVDEFENTRY, NameVals, + DefEntryAbbrev); NameVals.clear(); } + GUIDToValueIdMap.erase(VMI); + } + for (const auto &GVI : GUIDToValueIdMap) { + // VST_CODE_COMBINED_ENTRY: [valueid, refguid] + NameVals.push_back(GVI.second); + NameVals.push_back(GVI.first); + + // Emit the finished record. + Stream.EmitRecord(bitc::VST_CODE_COMBINED_ENTRY, NameVals, EntryAbbrev); + NameVals.clear(); } Stream.ExitBlock(); } @@ -2440,34 +2472,61 @@ static void WriteUseListBlock(const Function *F, ValueEnumerator &VE, Stream.ExitBlock(); } -/// \brief Save information for the given function into the function index. -/// -/// At a minimum this saves the bitcode index of the function record that -/// was just written. However, if we are emitting function summary information, -/// for example for ThinLTO, then a \a FunctionSummary object is created -/// to hold the provided summary information. -static void SaveFunctionInfo( - const Function &F, - DenseMap> &FunctionIndex, - unsigned NumInsts, uint64_t BitcodeIndex, bool EmitFunctionSummary) { - std::unique_ptr FuncSummary; - if (EmitFunctionSummary) { - FuncSummary = llvm::make_unique(NumInsts); - FuncSummary->setFunctionLinkage(F.getLinkage()); +// Walk through the operands of a given User via worklist iteration and populate +// the set of GlobalValue references encountered. Invoked either on an +// Instruction or a GlobalVariable (which walks its initializer). +static void findRefEdges(const User *CurUser, const ValueEnumerator &VE, + DenseSet &RefEdges, + SmallPtrSet &Visited) { + SmallVector Worklist; + Worklist.push_back(CurUser); + + while (!Worklist.empty()) { + const User *U = Worklist.pop_back_val(); + + if (!Visited.insert(U).second) + continue; + + ImmutableCallSite CS(U); + + for (const auto &OI : U->operands()) { + const User *Operand = dyn_cast(OI); + if (!Operand) + continue; + if (isa(Operand)) + continue; + if (isa(Operand)) { + // We have a reference to a global value. This should be added to + // the reference set unless it is a callee. Callees are handled + // specially by WriteFunction and are added to a separate list. + if (!(CS && CS.isCallee(&OI))) + RefEdges.insert(VE.getValueID(Operand)); + continue; + } + Worklist.push_back(Operand); + } } - FunctionIndex[&F] = - llvm::make_unique(BitcodeIndex, std::move(FuncSummary)); } /// Emit a function body to the module stream. static void WriteFunction( - const Function &F, ValueEnumerator &VE, BitstreamWriter &Stream, - DenseMap> &FunctionIndex, - bool EmitFunctionSummary) { + const Function &F, const Module *M, ValueEnumerator &VE, + BitstreamWriter &Stream, + DenseMap> &FunctionIndex, + bool EmitSummaryIndex) { // Save the bitcode index of the start of this function block for recording // in the VST. uint64_t BitcodeIndex = Stream.GetCurrentBitNo(); + bool HasProfileData = F.getEntryCount().hasValue(); + std::unique_ptr BFI; + if (EmitSummaryIndex && HasProfileData) { + Function &Func = const_cast(F); + LoopInfo LI{DominatorTree(Func)}; + BranchProbabilityInfo BPI{Func, LI}; + BFI = llvm::make_unique(Func, BPI, LI); + } + Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 4); VE.incorporateFunction(F); @@ -2494,7 +2553,12 @@ static void WriteFunction( DILocation *LastDL = nullptr; unsigned NumInsts = 0; + // Map from callee ValueId to profile count. Used to accumulate profile + // counts for all static calls to a given callee. + DenseMap CallGraphEdges; + DenseSet RefEdges; + SmallPtrSet Visited; // Finally, emit all the instructions, in order. for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB) for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); @@ -2507,6 +2571,24 @@ static void WriteFunction( if (!I->getType()->isVoidTy()) ++InstID; + if (EmitSummaryIndex) { + if (auto CS = ImmutableCallSite(&*I)) { + auto *CalledFunction = CS.getCalledFunction(); + if (CalledFunction && CalledFunction->hasName() && + !CalledFunction->isIntrinsic()) { + uint64_t ScaledCount = 0; + if (HasProfileData) + ScaledCount = getBlockProfileCount( + BFI->getBlockFreq(&(*BB)).getFrequency(), BFI->getEntryFreq(), + F.getEntryCount().getValue()); + unsigned CalleeId = VE.getValueID( + M->getValueSymbolTable().lookup(CalledFunction->getName())); + CallGraphEdges[CalleeId] += ScaledCount; + } + } + findRefEdges(&*I, VE, RefEdges, Visited); + } + // If the instruction has metadata, write a metadata attachment later. NeedsMetadataAttachment |= I->hasMetadataOtherThanDebugLoc(); @@ -2531,6 +2613,15 @@ static void WriteFunction( LastDL = DL; } + std::unique_ptr FuncSummary; + if (EmitSummaryIndex) { + FuncSummary = llvm::make_unique(F.getLinkage(), NumInsts); + FuncSummary->addCallGraphEdges(CallGraphEdges); + FuncSummary->addRefEdges(RefEdges); + } + FunctionIndex[&F] = + llvm::make_unique(BitcodeIndex, std::move(FuncSummary)); + // Emit names for all the instructions etc. WriteValueSymbolTable(F.getValueSymbolTable(), VE, Stream); @@ -2540,9 +2631,6 @@ static void WriteFunction( WriteUseListBlock(&F, VE, Stream); VE.purgeFunction(); Stream.ExitBlock(); - - SaveFunctionInfo(F, FunctionIndex, NumInsts, BitcodeIndex, - EmitFunctionSummary); } // Emit blockinfo, which defines the standard abbreviations etc. @@ -2776,34 +2864,103 @@ static void WriteModStrings(const FunctionInfoIndex &I, // Helper to emit a single function summary record. static void WritePerModuleFunctionSummaryRecord( - SmallVector &NameVals, FunctionSummary *FS, unsigned ValueID, - unsigned FSAbbrev, BitstreamWriter &Stream) { + SmallVector &NameVals, FunctionSummary *FS, unsigned ValueID, + unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev, + BitstreamWriter &Stream, const Function &F) { assert(FS); NameVals.push_back(ValueID); - NameVals.push_back(getEncodedLinkage(FS->getFunctionLinkage())); + NameVals.push_back(getEncodedLinkage(FS->linkage())); NameVals.push_back(FS->instCount()); + NameVals.push_back(FS->refs().size()); + + for (auto &RI : FS->refs()) + NameVals.push_back(RI); + + bool HasProfileData = F.getEntryCount().hasValue(); + for (auto &ECI : FS->edges()) { + NameVals.push_back(ECI.first); + assert(ECI.second.CallsiteCount > 0 && "Expected at least one callsite"); + NameVals.push_back(ECI.second.CallsiteCount); + if (HasProfileData) + NameVals.push_back(ECI.second.ProfileCount); + } + + unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); + unsigned Code = + (HasProfileData ? bitc::FS_PERMODULE_PROFILE : bitc::FS_PERMODULE); // Emit the finished record. - Stream.EmitRecord(bitc::FS_CODE_PERMODULE_ENTRY, NameVals, FSAbbrev); + Stream.EmitRecord(Code, NameVals, FSAbbrev); NameVals.clear(); } -/// Emit the per-module function summary section alongside the rest of +// Collect the global value references in the given variable's initializer, +// and emit them in a summary record. +static void WriteModuleLevelReferences(const GlobalVariable &V, + const ValueEnumerator &VE, + SmallVector &NameVals, + unsigned FSModRefsAbbrev, + BitstreamWriter &Stream) { + DenseSet RefEdges; + SmallPtrSet Visited; + findRefEdges(&V, VE, RefEdges, Visited); + unsigned RefCount = RefEdges.size(); + if (RefCount) { + NameVals.push_back(VE.getValueID(&V)); + NameVals.push_back(getEncodedLinkage(V.getLinkage())); + for (auto RefId : RefEdges) { + NameVals.push_back(RefId); + } + Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals, + FSModRefsAbbrev); + NameVals.clear(); + } +} + +/// Emit the per-module summary section alongside the rest of /// the module's bitcode. -static void WritePerModuleFunctionSummary( - DenseMap> &FunctionIndex, +static void WritePerModuleGlobalValueSummary( + DenseMap> &FunctionIndex, const Module *M, const ValueEnumerator &VE, BitstreamWriter &Stream) { - Stream.EnterSubblock(bitc::FUNCTION_SUMMARY_BLOCK_ID, 3); + if (M->empty()) + return; + + Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3); - // Abbrev for FS_CODE_PERMODULE_ENTRY. + // Abbrev for FS_PERMODULE. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_CODE_PERMODULE_ENTRY)); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount - unsigned FSAbbrev = Stream.EmitAbbrev(Abbv); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv); - SmallVector NameVals; + // Abbrev for FS_PERMODULE_PROFILE. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount, profilecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv); + + SmallVector NameVals; // Iterate over the list of functions instead of the FunctionIndex map to // ensure the ordering is stable. for (const Function &F : *M) { @@ -2817,9 +2974,9 @@ static void WritePerModuleFunctionSummary( assert(FunctionIndex.count(&F) == 1); WritePerModuleFunctionSummaryRecord( - NameVals, FunctionIndex[&F]->functionSummary(), - VE.getValueID(M->getValueSymbolTable().lookup(F.getName())), FSAbbrev, - Stream); + NameVals, dyn_cast(FunctionIndex[&F]->summary()), + VE.getValueID(M->getValueSymbolTable().lookup(F.getName())), + FSCallsAbbrev, FSCallsProfileAbbrev, Stream, F); } for (const GlobalAlias &A : M->aliases()) { @@ -2830,10 +2987,25 @@ static void WritePerModuleFunctionSummary( continue; assert(FunctionIndex.count(F) == 1); + FunctionSummary *FS = + dyn_cast(FunctionIndex[F]->summary()); + // Add the alias to the reference list of aliasee function. + FS->addRefEdge( + VE.getValueID(M->getValueSymbolTable().lookup(A.getName()))); WritePerModuleFunctionSummaryRecord( - NameVals, FunctionIndex[F]->functionSummary(), - VE.getValueID(M->getValueSymbolTable().lookup(A.getName())), FSAbbrev, - Stream); + NameVals, FS, + VE.getValueID(M->getValueSymbolTable().lookup(A.getName())), + FSCallsAbbrev, FSCallsProfileAbbrev, Stream, *F); + } + + // Capture references from GlobalVariable initializers, which are outside + // of a function scope. + for (const GlobalVariable &G : M->globals()) + WriteModuleLevelReferences(G, VE, NameVals, FSModRefsAbbrev, Stream); + for (const GlobalAlias &A : M->aliases()) { + const auto *GV = dyn_cast(A.getBaseObject()); + if (GV) + WriteModuleLevelReferences(*GV, VE, NameVals, FSModRefsAbbrev, Stream); } Stream.ExitBlock(); @@ -2841,35 +3013,132 @@ static void WritePerModuleFunctionSummary( /// Emit the combined function summary section into the combined index /// file. -static void WriteCombinedFunctionSummary(const FunctionInfoIndex &I, - BitstreamWriter &Stream) { - Stream.EnterSubblock(bitc::FUNCTION_SUMMARY_BLOCK_ID, 3); +static void WriteCombinedGlobalValueSummary( + const FunctionInfoIndex &I, BitstreamWriter &Stream, + std::map &GUIDToValueIdMap, unsigned GlobalValueId) { + Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3); - // Abbrev for FS_CODE_COMBINED_ENTRY. + // Abbrev for FS_COMBINED. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_CODE_COMBINED_ENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount - unsigned FSAbbrev = Stream.EmitAbbrev(Abbv); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv); - SmallVector NameVals; + // Abbrev for FS_COMBINED_PROFILE. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount, profilecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv); + + SmallVector NameVals; for (const auto &FII : I) { for (auto &FI : FII.second) { - FunctionSummary *FS = FI->functionSummary(); - assert(FS); + GlobalValueSummary *S = FI->summary(); + assert(S); + + if (auto *VS = dyn_cast(S)) { + assert(!VS->refs().empty() && "Expected at least one ref edge"); + NameVals.push_back(I.getModuleId(VS->modulePath())); + NameVals.push_back(getEncodedLinkage(VS->linkage())); + for (auto &RI : VS->refs()) { + const auto &VMI = GUIDToValueIdMap.find(RI); + unsigned RefId; + // If this GUID doesn't have an entry, assign one. + if (VMI == GUIDToValueIdMap.end()) { + GUIDToValueIdMap[RI] = ++GlobalValueId; + RefId = GlobalValueId; + } else { + RefId = VMI->second; + } + NameVals.push_back(RefId); + } + + // Record the starting offset of this summary entry for use + // in the VST entry. Add the current code size since the + // reader will invoke readRecord after the abbrev id read. + FI->setBitcodeIndex(Stream.GetCurrentBitNo() + + Stream.GetAbbrevIDWidth()); + + // Emit the finished record. + Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals, + FSModRefsAbbrev); + NameVals.clear(); + continue; + } + auto *FS = dyn_cast(S); + assert(FS); NameVals.push_back(I.getModuleId(FS->modulePath())); - NameVals.push_back(getEncodedLinkage(FS->getFunctionLinkage())); + NameVals.push_back(getEncodedLinkage(FS->linkage())); NameVals.push_back(FS->instCount()); + NameVals.push_back(FS->refs().size()); + + for (auto &RI : FS->refs()) { + const auto &VMI = GUIDToValueIdMap.find(RI); + unsigned RefId; + // If this GUID doesn't have an entry, assign one. + if (VMI == GUIDToValueIdMap.end()) { + GUIDToValueIdMap[RI] = ++GlobalValueId; + RefId = GlobalValueId; + } else { + RefId = VMI->second; + } + NameVals.push_back(RefId); + } + + bool HasProfileData = false; + for (auto &EI : FS->edges()) { + HasProfileData |= EI.second.ProfileCount != 0; + if (HasProfileData) + break; + } + + for (auto &EI : FS->edges()) { + const auto &VMI = GUIDToValueIdMap.find(EI.first); + // If this GUID doesn't have an entry, it doesn't have a function + // summary and we don't need to record any calls to it. + if (VMI == GUIDToValueIdMap.end()) + continue; + NameVals.push_back(VMI->second); + assert(EI.second.CallsiteCount > 0 && "Expected at least one callsite"); + NameVals.push_back(EI.second.CallsiteCount); + if (HasProfileData) + NameVals.push_back(EI.second.ProfileCount); + } // Record the starting offset of this summary entry for use // in the VST entry. Add the current code size since the // reader will invoke readRecord after the abbrev id read. FI->setBitcodeIndex(Stream.GetCurrentBitNo() + Stream.GetAbbrevIDWidth()); + unsigned FSAbbrev = + (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); + unsigned Code = + (HasProfileData ? bitc::FS_COMBINED_PROFILE : bitc::FS_COMBINED); + // Emit the finished record. - Stream.EmitRecord(bitc::FS_CODE_COMBINED_ENTRY, NameVals, FSAbbrev); + Stream.EmitRecord(Code, NameVals, FSAbbrev); NameVals.clear(); } } @@ -2904,7 +3173,7 @@ static void WriteIdentificationBlock(const Module *M, BitstreamWriter &Stream) { /// WriteModule - Emit the specified module to the bitstream. static void WriteModule(const Module *M, BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, - uint64_t BitcodeStartBit, bool EmitFunctionSummary) { + uint64_t BitcodeStartBit, bool EmitSummaryIndex) { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); SmallVector Vals; @@ -2949,15 +3218,15 @@ static void WriteModule(const Module *M, BitstreamWriter &Stream, WriteOperandBundleTags(M, Stream); // Emit function bodies. - DenseMap> FunctionIndex; + DenseMap> FunctionIndex; for (Module::const_iterator F = M->begin(), E = M->end(); F != E; ++F) if (!F->isDeclaration()) - WriteFunction(*F, VE, Stream, FunctionIndex, EmitFunctionSummary); + WriteFunction(*F, M, VE, Stream, FunctionIndex, EmitSummaryIndex); // Need to write after the above call to WriteFunction which populates // the summary information in the index. - if (EmitFunctionSummary) - WritePerModuleFunctionSummary(FunctionIndex, M, VE, Stream); + if (EmitSummaryIndex) + WritePerModuleGlobalValueSummary(FunctionIndex, M, VE, Stream); WriteValueSymbolTable(M->getValueSymbolTable(), VE, Stream, VSTOffsetPlaceholder, BitcodeStartBit, &FunctionIndex); @@ -3046,7 +3315,7 @@ static void WriteBitcodeHeader(BitstreamWriter &Stream) { /// stream. void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, bool ShouldPreserveUseListOrder, - bool EmitFunctionSummary) { + bool EmitSummaryIndex) { SmallVector Buffer; Buffer.reserve(256*1024); @@ -3072,7 +3341,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, // Emit the module. WriteModule(M, Stream, ShouldPreserveUseListOrder, BitcodeStartBit, - EmitFunctionSummary); + EmitSummaryIndex); } if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) @@ -3082,11 +3351,10 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, Out.write((char*)&Buffer.front(), Buffer.size()); } -// Write the specified function summary index to the given raw output stream, +// Write the specified module summary index to the given raw output stream, // where it will be written in a new bitcode block. This is used when // writing the combined index file for ThinLTO. -void llvm::WriteFunctionSummaryToFile(const FunctionInfoIndex &Index, - raw_ostream &Out) { +void llvm::WriteIndexToFile(const FunctionInfoIndex &Index, raw_ostream &Out) { SmallVector Buffer; Buffer.reserve(256 * 1024); @@ -3102,15 +3370,30 @@ void llvm::WriteFunctionSummaryToFile(const FunctionInfoIndex &Index, Vals.push_back(CurVersion); Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals); + // If we have a VST, write the VSTOFFSET record placeholder and record + // its offset. + uint64_t VSTOffsetPlaceholder = WriteValueSymbolTableForwardDecl(Stream); + // Write the module paths in the combined index. WriteModStrings(Index, Stream); - // Write the function summary combined index records. - WriteCombinedFunctionSummary(Index, Stream); + // Assign unique value ids to all functions in the index for use + // in writing out the call graph edges. Save the mapping from GUID + // to the new global value id to use when writing those edges, which + // are currently saved in the index in terms of GUID. + std::map GUIDToValueIdMap; + unsigned GlobalValueId = 0; + for (auto &II : Index) + GUIDToValueIdMap[II.first] = ++GlobalValueId; + + // Write the summary combined index records. + WriteCombinedGlobalValueSummary(Index, Stream, GUIDToValueIdMap, + GlobalValueId); // Need a special VST writer for the combined index (we don't have a // real VST and real values when this is invoked). - WriteCombinedValueSymbolTable(Index, Stream); + WriteCombinedValueSymbolTable(Index, Stream, GUIDToValueIdMap, + VSTOffsetPlaceholder); Stream.ExitBlock(); -- cgit v1.1