From 88fbc4d3df7c24105eb36232ff6250da95c8a202 Mon Sep 17 00:00:00 2001 From: Teresa Johnson Date: Wed, 6 Dec 2023 08:41:44 -0800 Subject: [ThinLTO] Add tail call flag to call edges in summary (#74043) This adds support for a HasTailCall flag on function call edges in the ThinLTO summary. It is intended for use in aiding discovery of missing frames from tail calls in profiled call stacks for MemProf of profiled binaries that did not disable tail call elimination. A follow on change will add the use of this new flag during MemProf context disambiguation. The new flag is encoded in the bitcode along with either the hotness flag from the profile, or the relative block frequency under the -write-relbf-to-summary flag when there is no profile data. Because we now will always have some additional call edge information, I have removed the non-profile function summary record format, and we simply encode the tail call flag along with a hotness type of none when there is no profile information or relative block frequency. The change of record format and name caused most of the test case changes. I have added explicit testing of generation of the new tail call flag into the bitcode and IR assembly format as part of the changes to llvm/test/Bitcode/thinlto-function-summary-refgraph.ll. I have also added round trip testing through assembly and bitcode to llvm/test/Assembler/thinlto-summary.ll. --- llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 34 +++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) (limited to 'llvm/lib/Bitcode/Reader/BitcodeReader.cpp') diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 1052bd4..9d7e838 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -1117,6 +1117,22 @@ static GlobalVarSummary::GVarFlags getDecodedGVarFlags(uint64_t RawFlags) { (GlobalObject::VCallVisibility)(RawFlags >> 3)); } +static std::pair +getDecodedHotnessCallEdgeInfo(uint64_t RawFlags) { + CalleeInfo::HotnessType Hotness = + static_cast(RawFlags & 0x7); // 3 bits + bool HasTailCall = (RawFlags & 0x8); // 1 bit + return {Hotness, HasTailCall}; +} + +static void getDecodedRelBFCallEdgeInfo(uint64_t RawFlags, uint64_t &RelBF, + bool &HasTailCall) { + static constexpr uint64_t RelBlockFreqMask = + (1 << CalleeInfo::RelBlockFreqBits) - 1; + RelBF = RawFlags & RelBlockFreqMask; // RelBlockFreqBits bits + HasTailCall = (RawFlags & (1 << CalleeInfo::RelBlockFreqBits)); // 1 bit +} + static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) { switch (Val) { default: // Map unknown visibilities to default. @@ -7032,6 +7048,7 @@ ModuleSummaryIndexBitcodeReader::makeCallList(ArrayRef Record, Ret.reserve(Record.size()); for (unsigned I = 0, E = Record.size(); I != E; ++I) { CalleeInfo::HotnessType Hotness = CalleeInfo::HotnessType::Unknown; + bool HasTailCall = false; uint64_t RelBF = 0; ValueInfo Callee = std::get<0>(getValueInfoFromValueId(Record[I])); if (IsOldProfileFormat) { @@ -7039,10 +7056,12 @@ ModuleSummaryIndexBitcodeReader::makeCallList(ArrayRef Record, if (HasProfile) I += 1; // Skip old profilecount field } else if (HasProfile) - Hotness = static_cast(Record[++I]); + std::tie(Hotness, HasTailCall) = + getDecodedHotnessCallEdgeInfo(Record[++I]); else if (HasRelBF) - RelBF = Record[++I]; - Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo(Hotness, RelBF)}); + getDecodedRelBFCallEdgeInfo(Record[++I], RelBF, HasTailCall); + Ret.push_back(FunctionSummary::EdgeTy{ + Callee, CalleeInfo(Hotness, HasTailCall, RelBF)}); } return Ret; } @@ -7256,14 +7275,15 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { TheIndex.getOrInsertValueInfo(RefGUID), RefGUID, RefGUID); break; } + // FS_PERMODULE is legacy and does not have support for the tail call flag. // FS_PERMODULE: [valueid, flags, instcount, fflags, numrefs, // numrefs x valueid, n x (valueid)] // FS_PERMODULE_PROFILE: [valueid, flags, instcount, fflags, numrefs, // numrefs x valueid, - // n x (valueid, hotness)] + // n x (valueid, hotness+tailcall flags)] // FS_PERMODULE_RELBF: [valueid, flags, instcount, fflags, numrefs, // numrefs x valueid, - // n x (valueid, relblockfreq)] + // n x (valueid, relblockfreq+tailcall)] case bitc::FS_PERMODULE: case bitc::FS_PERMODULE_RELBF: case bitc::FS_PERMODULE_PROFILE: { @@ -7410,10 +7430,12 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { TheIndex.addGlobalValueSummary(std::get<0>(GUID), std::move(VS)); break; } + // FS_COMBINED is legacy and does not have support for the tail call flag. // FS_COMBINED: [valueid, modid, flags, instcount, fflags, numrefs, // numrefs x valueid, n x (valueid)] // FS_COMBINED_PROFILE: [valueid, modid, flags, instcount, fflags, numrefs, - // numrefs x valueid, n x (valueid, hotness)] + // numrefs x valueid, + // n x (valueid, hotness+tailcall flags)] case bitc::FS_COMBINED: case bitc::FS_COMBINED_PROFILE: { unsigned ValueID = Record[0]; -- cgit v1.1