aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/ProfileData
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/ProfileData')
-rw-r--r--llvm/lib/ProfileData/SampleProf.cpp43
-rw-r--r--llvm/lib/ProfileData/SampleProfReader.cpp98
-rw-r--r--llvm/lib/ProfileData/SampleProfWriter.cpp78
3 files changed, 210 insertions, 9 deletions
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index 60c1393..f870618 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -47,6 +47,24 @@ bool FunctionSamples::ProfileIsPreInlined = false;
bool FunctionSamples::UseMD5 = false;
bool FunctionSamples::HasUniqSuffix = true;
bool FunctionSamples::ProfileIsFS = false;
+
+std::error_code
+serializeTypeMap(const TypeCountMap &Map,
+ const MapVector<FunctionId, uint32_t> &NameTable,
+ raw_ostream &OS) {
+ encodeULEB128(Map.size(), OS);
+ for (const auto &[TypeName, SampleCount] : Map) {
+ if (auto NameIndexIter = NameTable.find(TypeName);
+ NameIndexIter != NameTable.end()) {
+ encodeULEB128(NameIndexIter->second, OS);
+ } else {
+ // If the type is not in the name table, we cannot serialize it.
+ return sampleprof_error::truncated_name_table;
+ }
+ encodeULEB128(SampleCount, OS);
+ }
+ return sampleprof_error::success;
+}
} // namespace sampleprof
} // namespace llvm
@@ -93,6 +111,8 @@ class SampleProfErrorCategoryType : public std::error_category {
return "Function hash mismatch";
case sampleprof_error::illegal_line_offset:
return "Illegal line offset in sample profile data";
+ case sampleprof_error::duplicate_vtable_type:
+ return "Duplicate vtable type in one map";
}
llvm_unreachable("A value of sampleprof_error has no message.");
}
@@ -126,6 +146,7 @@ sampleprof_error SampleRecord::merge(const SampleRecord &Other,
for (const auto &I : Other.getCallTargets()) {
mergeSampleProfErrors(Result, addCalledTarget(I.first, I.second, Weight));
}
+
return Result;
}
@@ -178,6 +199,17 @@ raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
return OS;
}
+static void printTypeCountMap(raw_ostream &OS, LineLocation Loc,
+ const TypeCountMap &TypeCountMap) {
+ if (TypeCountMap.empty()) {
+ return;
+ }
+ OS << Loc << ": vtables: ";
+ for (const auto &[Type, Count] : TypeCountMap)
+ OS << Type << ":" << Count << " ";
+ OS << "\n";
+}
+
/// Print the samples collected for a function on stream \p OS.
void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
if (getFunctionHash())
@@ -192,7 +224,13 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
SampleSorter<LineLocation, SampleRecord> SortedBodySamples(BodySamples);
for (const auto &SI : SortedBodySamples.get()) {
OS.indent(Indent + 2);
+ const auto &Loc = SI->first;
OS << SI->first << ": " << SI->second;
+ if (const TypeCountMap *TypeCountMap =
+ this->findCallsiteTypeSamplesAt(Loc)) {
+ OS.indent(Indent + 2);
+ printTypeCountMap(OS, Loc, *TypeCountMap);
+ }
}
OS.indent(Indent);
OS << "}\n";
@@ -214,6 +252,11 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
OS << Loc << ": inlined callee: " << FuncSample.getFunction() << ": ";
FuncSample.print(OS, Indent + 4);
}
+ auto TypeSamplesIter = VirtualCallsiteTypeCounts.find(Loc);
+ if (TypeSamplesIter != VirtualCallsiteTypeCounts.end()) {
+ OS.indent(Indent + 2);
+ printTypeCountMap(OS, Loc, TypeSamplesIter->second);
+ }
}
OS.indent(Indent);
OS << "}\n";
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index cf78740..b8b55f7 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -197,8 +197,30 @@ enum class LineType {
CallSiteProfile,
BodyProfile,
Metadata,
+ VirtualCallTypeProfile,
};
+static bool parseTypeCountMap(StringRef Input,
+ DenseMap<StringRef, uint64_t> &TypeCountMap) {
+ for (size_t Index = Input.find_first_not_of(' '); Index != StringRef::npos;) {
+ size_t n1 = Input.find(':', Index);
+ if (n1 == StringRef::npos)
+ return false; // No colon found, invalid format.
+ StringRef TypeName = Input.substr(Index, n1 - Index);
+ // n2 is the start index of count.
+ size_t n2 = n1 + 1;
+ // n3 is the start index after the 'target:count' pair.
+ size_t n3 = Input.find_first_of(' ', n2);
+ uint64_t Count;
+ if (Input.substr(n2, n3 - n2).getAsInteger(10, Count))
+ return false; // Invalid count.
+ TypeCountMap[TypeName] = Count;
+ Index = (n3 == StringRef::npos) ? StringRef::npos
+ : Input.find_first_not_of(' ', n3);
+ }
+ return true;
+}
+
/// Parse \p Input as line sample.
///
/// \param Input input line.
@@ -215,6 +237,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
uint64_t &NumSamples, uint32_t &LineOffset,
uint32_t &Discriminator, StringRef &CalleeName,
DenseMap<StringRef, uint64_t> &TargetCountMap,
+ DenseMap<StringRef, uint64_t> &TypeCountMap,
uint64_t &FunctionHash, uint32_t &Attributes,
bool &IsFlat) {
for (Depth = 0; Input[Depth] == ' '; Depth++)
@@ -289,6 +312,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
n4 = AfterColon.find_first_of(' ');
n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1);
+ // Break the loop if parsing integer succeeded.
if (!WordAfterColon.getAsInteger(10, count))
break;
@@ -306,6 +330,10 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
// Change n3 to the next blank space after colon + integer pair.
n3 = n4;
}
+ } else if (Rest.starts_with(kVTableProfPrefix)) {
+ LineTy = LineType::VirtualCallTypeProfile;
+ return parseTypeCountMap(Rest.substr(strlen(kVTableProfPrefix)),
+ TypeCountMap);
} else {
LineTy = LineType::CallSiteProfile;
size_t n3 = Rest.find_last_of(':');
@@ -374,14 +402,15 @@ std::error_code SampleProfileReaderText::readImpl() {
uint64_t NumSamples;
StringRef FName;
DenseMap<StringRef, uint64_t> TargetCountMap;
+ DenseMap<StringRef, uint64_t> TypeCountMap;
uint32_t Depth, LineOffset, Discriminator;
LineType LineTy;
uint64_t FunctionHash = 0;
uint32_t Attributes = 0;
bool IsFlat = false;
if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset,
- Discriminator, FName, TargetCountMap, FunctionHash,
- Attributes, IsFlat)) {
+ Discriminator, FName, TargetCountMap, TypeCountMap,
+ FunctionHash, Attributes, IsFlat)) {
reportError(LineIt.line_number(),
"Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
*LineIt);
@@ -410,6 +439,14 @@ std::error_code SampleProfileReaderText::readImpl() {
DepthMetadata = 0;
break;
}
+
+ case LineType::VirtualCallTypeProfile: {
+ mergeSampleProfErrors(
+ Result, InlineStack.back()->addCallsiteVTableTypeProfAt(
+ LineLocation(LineOffset, Discriminator), TypeCountMap));
+ break;
+ }
+
case LineType::BodyProfile: {
FunctionSamples &FProfile = *InlineStack.back();
for (const auto &name_count : TargetCountMap) {
@@ -592,6 +629,59 @@ SampleProfileReaderBinary::readSampleContextFromTable() {
}
std::error_code
+SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) {
+ auto NumVTableTypes = readNumber<uint32_t>();
+ if (std::error_code EC = NumVTableTypes.getError())
+ return EC;
+
+ for (uint32_t I = 0; I < *NumVTableTypes; ++I) {
+ auto VTableType(readStringFromTable());
+ if (std::error_code EC = VTableType.getError())
+ return EC;
+
+ auto VTableSamples = readNumber<uint64_t>();
+ if (std::error_code EC = VTableSamples.getError())
+ return EC;
+
+ if (!M.insert(std::make_pair(*VTableType, *VTableSamples)).second)
+ return sampleprof_error::duplicate_vtable_type;
+ }
+ return sampleprof_error::success;
+}
+
+std::error_code
+SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) {
+ if (!ReadVTableProf)
+ return sampleprof_error::success;
+
+ // Read the vtable type profile for the callsite.
+ auto NumCallsites = readNumber<uint32_t>();
+ if (std::error_code EC = NumCallsites.getError())
+ return EC;
+
+ for (uint32_t I = 0; I < *NumCallsites; ++I) {
+ auto LineOffset = readNumber<uint64_t>();
+ if (std::error_code EC = LineOffset.getError())
+ return EC;
+
+ if (!isOffsetLegal(*LineOffset))
+ return sampleprof_error::illegal_line_offset;
+
+ auto Discriminator = readNumber<uint64_t>();
+ if (std::error_code EC = Discriminator.getError())
+ return EC;
+
+ // Here we handle FS discriminators:
+ const uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
+
+ if (std::error_code EC = readVTableTypeCountMap(FProfile.getTypeSamplesAt(
+ LineLocation(*LineOffset, DiscriminatorVal))))
+ return EC;
+ }
+ return sampleprof_error::success;
+}
+
+std::error_code
SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
auto NumSamples = readNumber<uint64_t>();
if (std::error_code EC = NumSamples.getError())
@@ -671,7 +761,7 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
return EC;
}
- return sampleprof_error::success;
+ return readCallsiteVTableProf(FProfile);
}
std::error_code
@@ -733,6 +823,8 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;
if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator))
FunctionSamples::ProfileIsFS = ProfileIsFS = true;
+ if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagHasVTableTypeProf))
+ ReadVTableProf = true;
break;
case SecNameTable: {
bool FixedLengthMD5 =
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index 9173a0f..dfab404 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -41,6 +41,11 @@
using namespace llvm;
using namespace sampleprof;
+// To begin with, make this option off by default.
+static cl::opt<bool> ExtBinaryWriteVTableTypeProf(
+ "extbinary-write-vtable-type-prof", cl::init(false), cl::Hidden,
+ cl::desc("Write vtable type profile in ext-binary sample profile writer"));
+
namespace llvm {
namespace support {
namespace endian {
@@ -435,6 +440,9 @@ std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagIsPreInlined);
if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)
addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFSDiscriminator);
+ if (Type == SecProfSummary && ExtBinaryWriteVTableTypeProf)
+ addSectionFlag(SecProfSummary,
+ SecProfSummaryFlags::SecFlagHasVTableTypeProf);
uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
switch (Type) {
@@ -478,6 +486,12 @@ std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
return sampleprof_error::success;
}
+SampleProfileWriterExtBinary::SampleProfileWriterExtBinary(
+ std::unique_ptr<raw_ostream> &OS)
+ : SampleProfileWriterExtBinaryBase(OS) {
+ WriteVTableProf = ExtBinaryWriteVTableTypeProf;
+}
+
std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
const SampleProfileMap &ProfileMap) {
// The const indices passed to writeOneSection below are specifying the
@@ -587,6 +601,19 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
OS << " " << J.first << ":" << J.second;
OS << "\n";
LineCount++;
+
+ if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
+ Map && !Map->empty()) {
+ OS.indent(Indent + 1);
+ Loc.print(OS);
+ OS << ": ";
+ OS << kVTableProfPrefix;
+ for (const auto [TypeName, Count] : *Map) {
+ OS << TypeName << ":" << Count << " ";
+ }
+ OS << "\n";
+ LineCount++;
+ }
}
SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
@@ -603,7 +630,21 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
if (std::error_code EC = writeSample(CalleeSamples))
return EC;
}
+
+ if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
+ Map && !Map->empty()) {
+ OS.indent(Indent);
+ Loc.print(OS);
+ OS << ": ";
+ OS << kVTableProfPrefix;
+ for (const auto [TypeId, Count] : *Map) {
+ OS << TypeId << ":" << Count << " ";
+ }
+ OS << "\n";
+ LineCount++;
+ }
}
+
Indent -= 1;
if (FunctionSamples::ProfileIsProbeBased) {
@@ -663,6 +704,17 @@ void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
addName(CalleeSamples.getFunction());
addNames(CalleeSamples);
}
+
+ if (!WriteVTableProf)
+ return;
+ // Add all the vtable names to NameTable.
+ for (const auto &VTableAccessCountMap :
+ llvm::make_second_range(S.getCallsiteTypeCounts())) {
+ // Add type name to NameTable.
+ for (const auto Type : llvm::make_first_range(VTableAccessCountMap)) {
+ addName(Type);
+ }
+ }
}
void SampleProfileWriterExtBinaryBase::addContext(
@@ -801,6 +853,21 @@ std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
return sampleprof_error::success;
}
+std::error_code SampleProfileWriterBinary::writeCallsiteVTableProf(
+ const CallsiteTypeMap &CallsiteTypeMap, raw_ostream &OS) {
+ if (!WriteVTableProf)
+ return sampleprof_error::success;
+
+ encodeULEB128(CallsiteTypeMap.size(), OS);
+ for (const auto &[Loc, TypeMap] : CallsiteTypeMap) {
+ Loc.serialize(OS);
+ if (std::error_code EC = serializeTypeMap(TypeMap, getNameTable(), OS))
+ return EC;
+ }
+
+ return sampleprof_error::success;
+}
+
std::error_code SampleProfileWriterBinary::writeSummary() {
auto &OS = *OutputStream;
encodeULEB128(Summary->getTotalCount(), OS);
@@ -838,15 +905,14 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
for (const auto &J : S.getCallsiteSamples())
NumCallsites += J.second.size();
encodeULEB128(NumCallsites, OS);
- for (const auto &[Loc, CalleeFunctionSampleMap] : S.getCallsiteSamples())
- for (const auto &FunctionSample :
- llvm::make_second_range(CalleeFunctionSampleMap)) {
- Loc.serialize(OS);
- if (std::error_code EC = writeBody(FunctionSample))
+ for (const auto &J : S.getCallsiteSamples())
+ for (const auto &FS : J.second) {
+ J.first.serialize(OS);
+ if (std::error_code EC = writeBody(FS.second))
return EC;
}
- return sampleprof_error::success;
+ return writeCallsiteVTableProf(S.getCallsiteTypeCounts(), OS);
}
/// Write samples of a top-level function to a binary file.
generated by cgit v1.1 at 2025-09-28 03:59:12 +0000