diff options
author | Mingming Liu <mingmingl@google.com> | 2024-04-01 15:14:49 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-04-01 15:14:49 -0700 |
commit | 1e15371dd8843dfc52b9435afaa133997c1773d8 (patch) | |
tree | 7341201ff5cbb4601b29dd8d9db70c650f3f2c1e | |
parent | 1d5e5f4d3c68e63ced47ee9b17d62fb995aa1e62 (diff) | |
download | llvm-1e15371dd8843dfc52b9435afaa133997c1773d8.zip llvm-1e15371dd8843dfc52b9435afaa133997c1773d8.tar.gz llvm-1e15371dd8843dfc52b9435afaa133997c1773d8.tar.bz2 |
[ThinLTO][TypeProf] Implement vtable def import (#79381)
Add annotated vtable GUID as referenced variables in per function
summary, and update bitcode writer to create value-ids for these
referenced vtables.
- This is the part3 of type profiling work, and described in the "Virtual Table Definition Import" [1] section of the
RFC.
[1] https://github.com/llvm/llvm-project/pull/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
-rw-r--r-- | llvm/include/llvm/ProfileData/InstrProf.h | 12 | ||||
-rw-r--r-- | llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp | 4 | ||||
-rw-r--r-- | llvm/lib/Analysis/ModuleSummaryAnalysis.cpp | 20 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 13 | ||||
-rw-r--r-- | llvm/lib/ProfileData/InstrProf.cpp | 70 | ||||
-rw-r--r-- | llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll | 37 |
6 files changed, 120 insertions, 36 deletions
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index fd66c4e..eb3c10b 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -283,7 +283,7 @@ void annotateValueSite(Module &M, Instruction &Inst, /// Extract the value profile data from \p Inst which is annotated with /// value profile meta data. Return false if there is no value data annotated, -/// otherwise return true. +/// otherwise return true. bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, @@ -291,6 +291,16 @@ bool getValueProfDataFromInst(const Instruction &Inst, uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue = false); +/// Extract the value profile data from \p Inst and returns them if \p Inst is +/// annotated with value profile data. Returns nullptr otherwise. It's similar +/// to `getValueProfDataFromInst` above except that an array is allocated only +/// after a preliminary checking that the value profiles of kind `ValueKind` +/// exist. +std::unique_ptr<InstrProfValueData[]> +getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, uint32_t &ActualNumValueData, + uint64_t &TotalC, bool GetNoICPValue = false); + inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; } /// Return the PGOFuncName meta data associated with a function. diff --git a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp index ebfa1c8..ab53717 100644 --- a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp +++ b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp @@ -45,6 +45,10 @@ static cl::opt<unsigned> cl::desc("Max number of promotions for a single indirect " "call callsite")); +cl::opt<unsigned> MaxNumVTableAnnotations( + "icp-max-num-vtables", cl::init(6), cl::Hidden, + cl::desc("Max number of vtables annotated for a vtable load instruction.")); + ICallPromotionAnalysis::ICallPromotionAnalysis() { ValueDataArray = std::make_unique<InstrProfValueData[]>(MaxNumPromotions); } diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index 1f15e94..3ad0bab 100644 --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -82,6 +82,8 @@ static cl::opt<std::string> ModuleSummaryDotFile( extern cl::opt<bool> ScalePartialSampleProfileWorkingSetSize; +extern cl::opt<unsigned> MaxNumVTableAnnotations; + // Walk through the operands of a given User via worklist iteration and populate // the set of GlobalValue references encountered. Invoked either on an // Instruction or a GlobalVariable (which walks its initializer). @@ -124,6 +126,24 @@ static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser, Worklist.push_back(Operand); } } + + const Instruction *I = dyn_cast<Instruction>(CurUser); + if (I) { + uint32_t ActualNumValueData = 0; + uint64_t TotalCount = 0; + // MaxNumVTableAnnotations is the maximum number of vtables annotated on + // the instruction. + auto ValueDataArray = + getValueProfDataFromInst(*I, IPVK_VTableTarget, MaxNumVTableAnnotations, + ActualNumValueData, TotalCount); + + if (ValueDataArray.get()) { + for (uint32_t j = 0; j < ActualNumValueData; j++) { + RefEdges.insert(Index.getOrInsertValueInfo(/* VTableGUID = */ + ValueDataArray[j].Value)); + } + } + } return HasBlockAddress; } diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 221eeaa..dd554e4 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -203,7 +203,7 @@ public: for (const auto &GUIDSummaryLists : *Index) // Examine all summaries for this GUID. for (auto &Summary : GUIDSummaryLists.second.SummaryList) - if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) + if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) { // For each call in the function summary, see if the call // is to a GUID (which means it is for an indirect call, // otherwise we would have a Value for it). If so, synthesize @@ -211,6 +211,15 @@ public: for (auto &CallEdge : FS->calls()) if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue()) assignValueId(CallEdge.first.getGUID()); + + // For each referenced variables in the function summary, see if the + // variable is represented by a GUID (as opposed to a symbol to + // declarations or definitions in the module). If so, synthesize a + // value id. + for (auto &RefEdge : FS->refs()) + if (!RefEdge.haveGVs() || !RefEdge.getValue()) + assignValueId(RefEdge.getGUID()); + } } protected: @@ -4188,7 +4197,7 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( NameVals.push_back(SpecialRefCnts.second); // worefcnt for (auto &RI : FS->refs()) - NameVals.push_back(VE.getValueID(RI.getValue())); + NameVals.push_back(getValueId(RI)); const bool UseRelBFRecord = WriteRelBFToSummary && !F.hasProfileData() && diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 90c3cfc..95f900d0 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -1271,46 +1271,44 @@ void annotateValueSite(Module &M, Instruction &Inst, Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals)); } -bool getValueProfDataFromInst(const Instruction &Inst, - InstrProfValueKind ValueKind, - uint32_t MaxNumValueData, - InstrProfValueData ValueData[], - uint32_t &ActualNumValueData, uint64_t &TotalC, - bool GetNoICPValue) { +MDNode *mayHaveValueProfileOfKind(const Instruction &Inst, + InstrProfValueKind ValueKind) { MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof); if (!MD) - return false; + return nullptr; - unsigned NOps = MD->getNumOperands(); + if (MD->getNumOperands() < 5) + return nullptr; - if (NOps < 5) - return false; - - // Operand 0 is a string tag "VP": MDString *Tag = cast<MDString>(MD->getOperand(0)); - if (!Tag) - return false; - - if (!Tag->getString().equals("VP")) - return false; + if (!Tag || !Tag->getString().equals("VP")) + return nullptr; // Now check kind: ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1)); if (!KindInt) - return false; + return nullptr; if (KindInt->getZExtValue() != ValueKind) - return false; + return nullptr; + + return MD; +} +static bool getValueProfDataFromInstImpl(const MDNode *const MD, + const uint32_t MaxNumDataWant, + InstrProfValueData ValueData[], + uint32_t &ActualNumValueData, + uint64_t &TotalC, bool GetNoICPValue) { + const unsigned NOps = MD->getNumOperands(); // Get total count ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2)); if (!TotalCInt) return false; TotalC = TotalCInt->getZExtValue(); - ActualNumValueData = 0; for (unsigned I = 3; I < NOps; I += 2) { - if (ActualNumValueData >= MaxNumValueData) + if (ActualNumValueData >= MaxNumDataWant) break; ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I)); ConstantInt *Count = @@ -1327,6 +1325,36 @@ bool getValueProfDataFromInst(const Instruction &Inst, return true; } +std::unique_ptr<InstrProfValueData[]> +getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, uint32_t &ActualNumValueData, + uint64_t &TotalC, bool GetNoICPValue) { + MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind); + if (!MD) + return nullptr; + auto ValueDataArray = std::make_unique<InstrProfValueData[]>(MaxNumValueData); + if (!getValueProfDataFromInstImpl(MD, MaxNumValueData, ValueDataArray.get(), + ActualNumValueData, TotalC, GetNoICPValue)) + return nullptr; + return ValueDataArray; +} + +// FIXME: Migrate existing callers to the function above that returns an +// array. +bool getValueProfDataFromInst(const Instruction &Inst, + InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, + InstrProfValueData ValueData[], + uint32_t &ActualNumValueData, uint64_t &TotalC, + bool GetNoICPValue) { + MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind); + if (!MD) + return false; + return getValueProfDataFromInstImpl(MD, MaxNumValueData, ValueData, + ActualNumValueData, TotalC, + GetNoICPValue); +} + MDNode *getPGOFuncNameMetadata(const Function &F) { return F.getMetadata(getPGOFuncNameMetadataName()); } diff --git a/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll index 78b175c..ba3ce9a 100644 --- a/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll +++ b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll @@ -1,20 +1,31 @@ -; RUN: opt -module-summary %s -o %t.o +; Promote at most one function and annotate at most one vtable. +; As a result, only one value (of each relevant kind) shows up in the function +; summary. + +; RUN: opt -module-summary -icp-max-num-vtables=1 -icp-max-prom=1 %s -o %t.o ; RUN: llvm-bcanalyzer -dump %t.o | FileCheck %s ; RUN: llvm-dis -o - %t.o | FileCheck %s --check-prefix=DIS - +; Round trip it through llvm-as +; RUN: llvm-dis -o - %t.o | llvm-as -o - | llvm-dis -o - | FileCheck %s --check-prefix=DIS ; CHECK: <GLOBALVAL_SUMMARY_BLOCK ; CHECK-NEXT: <VERSION op0=9/> ; CHECK-NEXT: <FLAGS op0=0/> +; The `VALUE_GUID` below represents the "_ZTV4Base" referenced by the instruction +; that loads vtable pointers. +; CHECK-NEXT: <VALUE_GUID op0=21 op1=1960855528937986108/> ; The `VALUE_GUID` below represents the "_ZN4Base4funcEv" referenced by the ; indirect call instruction. -; CHECK-NEXT: <VALUE_GUID op0=17 op1=5459407273543877811/> +; CHECK-NEXT: <VALUE_GUID op0=20 op1=5459407273543877811/> +; NOTE vtables and functions from Derived class is dropped because +; `-icp-max-num-vtables` and `-icp-max-prom` are both set to one. ; <PERMODULE_PROFILE> has the format [valueid, flags, instcount, funcflags, ; numrefs, rorefcnt, worefcnt, +; m x valueid, ; n x (valueid, hotness+tailcall)] -; CHECK-NEXT: <PERMODULE_PROFILE abbrevid=4 op0=0 op1=0 op2=4 op3=256 op4=0 op5=0 op6=0 op7=17 op8=3/> +; CHECK-NEXT: <PERMODULE_PROFILE abbrevid=4 op0=0 op1=0 op2=4 op3=256 op4=1 op5=1 op6=0 op7=21 op8=20 op9=3/> ; CHECK-NEXT: </GLOBALVAL_SUMMARY_BLOCK> target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" @@ -31,7 +42,6 @@ define i32 @_Z4testP4Base(ptr %0) !prof !15 { !llvm.module.flags = !{!1} - !1 = !{i32 1, !"ProfileSummary", !2} !2 = !{!3, !4, !5, !6, !7, !8, !9, !10} !3 = !{!"ProfileFormat", !"InstrProf"} @@ -48,14 +58,17 @@ define i32 @_Z4testP4Base(ptr %0) !prof !15 { !14 = !{i32 999999, i64 1, i32 2} !15 = !{!"function_entry_count", i32 150} -; 1960855528937986108 is the MD5 hash of _ZTV4Base -!16 = !{!"VP", i32 2, i64 1600, i64 1960855528937986108, i64 1600} -; 5459407273543877811 is the MD5 hash of _ZN4Base4funcEv -!17 = !{!"VP", i32 0, i64 1600, i64 5459407273543877811, i64 1600} +; 1960855528937986108 is the MD5 hash of _ZTV4Base, and +; 13870436605473471591 is the MD5 hash of _ZTV7Derived +!16 = !{!"VP", i32 2, i64 150, i64 1960855528937986108, i64 100, i64 13870436605473471591, i64 50} +; 5459407273543877811 is the MD5 hash of _ZN4Base4funcEv, and +; 6174874150489409711 is the MD5 hash of _ZN7Derived4funcEv +!17 = !{!"VP", i32 0, i64 150, i64 5459407273543877811, i64 100, i64 6174874150489409711, i64 50} ; ModuleSummaryIndex stores <guid, global-value summary> map in std::map; so ; global value summares are printed out in the order that gv's guid increases. ; DIS: ^0 = module: (path: "{{.*}}", hash: (0, 0, 0, 0, 0)) -; DIS: ^1 = gv: (guid: 5459407273543877811) -; DIS: ^2 = gv: (name: "_Z4testP4Base", summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 4, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 1, mustBeUnreachable: 0), calls: ((callee: ^1, hotness: hot))))) ; guid = 15857150948103218965 -; DIS: ^3 = blockcount: 0 +; DIS: ^1 = gv: (guid: 1960855528937986108) +; DIS: ^2 = gv: (guid: 5459407273543877811) +; DIS: ^3 = gv: (name: "_Z4testP4Base", summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 4, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 1, mustBeUnreachable: 0), calls: ((callee: ^2, hotness: hot)), refs: (readonly ^1)))) ; guid = 15857150948103218965 +; DIS: ^4 = blockcount: 0 |