1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
#include "llvm/Analysis/StaticDataProfileInfo.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/InitializePasses.h"
#include "llvm/ProfileData/InstrProf.h"
using namespace llvm;
extern cl::opt<bool> AnnotateStaticDataSectionPrefix;
void StaticDataProfileInfo::addConstantProfileCount(
const Constant *C, std::optional<uint64_t> Count) {
if (!Count) {
ConstantWithoutCounts.insert(C);
return;
}
uint64_t &OriginalCount = ConstantProfileCounts[C];
OriginalCount = llvm::SaturatingAdd(*Count, OriginalCount);
// Clamp the count to getInstrMaxCountValue. InstrFDO reserves a few
// large values for special use.
if (OriginalCount > getInstrMaxCountValue())
OriginalCount = getInstrMaxCountValue();
}
std::optional<uint64_t>
StaticDataProfileInfo::getConstantProfileCount(const Constant *C) const {
auto I = ConstantProfileCounts.find(C);
if (I == ConstantProfileCounts.end())
return std::nullopt;
return I->second;
}
StaticDataProfileInfo::StaticDataHotness
StaticDataProfileInfo::getSectionHotnessUsingProfileCount(
const Constant *C, const ProfileSummaryInfo *PSI, uint64_t Count) const {
// The accummulated counter shows the constant is hot. Return 'hot' whether
// this variable is seen by unprofiled functions or not.
if (PSI->isHotCount(Count))
return StaticDataHotness::Hot;
// The constant is not hot, and seen by unprofiled functions. We don't want to
// assign it to unlikely sections, even if the counter says 'cold'. So return
// an empty prefix before checking whether the counter is cold.
if (ConstantWithoutCounts.count(C))
return StaticDataHotness::LukewarmOrUnknown;
// The accummulated counter shows the constant is cold. Return 'unlikely'.
if (PSI->isColdCount(Count))
return StaticDataHotness::Cold;
return StaticDataHotness::LukewarmOrUnknown;
}
StringRef StaticDataProfileInfo::hotnessToStr(
StaticDataProfileInfo::StaticDataHotness Hotness) const {
switch (Hotness) {
case StaticDataProfileInfo::StaticDataHotness::Cold:
return "unlikely";
case StaticDataProfileInfo::StaticDataHotness::Hot:
return "hot";
default:
return "";
}
}
StaticDataProfileInfo::StaticDataHotness
StaticDataProfileInfo::getSectionHotnessUsingDAP(
std::optional<StringRef> MaybeSectionPrefix) const {
if (!MaybeSectionPrefix)
return StaticDataProfileInfo::StaticDataHotness::LukewarmOrUnknown;
StringRef Prefix = *MaybeSectionPrefix;
assert((Prefix == "hot" || Prefix == "unlikely") &&
"Expect section_prefix to be one of hot or unlikely");
return Prefix == "hot" ? StaticDataProfileInfo::StaticDataHotness::Hot
: StaticDataProfileInfo::StaticDataHotness::Cold;
}
StringRef StaticDataProfileInfo::getConstantSectionPrefix(
const Constant *C, const ProfileSummaryInfo *PSI) const {
auto Count = getConstantProfileCount(C);
if (HasDataAccessProf) {
// Module flag `HasDataAccessProf` is 1 -> empty section prefix means uknown
// hotness except for string literals.
if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C);
GV && !GV->getName().starts_with(".str")) {
auto HotnessFromDAP = getSectionHotnessUsingDAP(GV->getSectionPrefix());
if (!Count)
return hotnessToStr(HotnessFromDAP);
auto HotnessFromPGO = getSectionHotnessUsingProfileCount(C, PSI, *Count);
return hotnessToStr(std::max(HotnessFromDAP, HotnessFromPGO));
}
}
if (!Count)
return "";
return hotnessToStr(getSectionHotnessUsingProfileCount(C, PSI, *Count));
}
bool StaticDataProfileInfoWrapperPass::doInitialization(Module &M) {
bool HasDataAccessProf = false;
if (auto *MD = mdconst::extract_or_null<ConstantInt>(
M.getModuleFlag("HasDataAccessProf")))
HasDataAccessProf = MD->getZExtValue();
Info.reset(new StaticDataProfileInfo(HasDataAccessProf));
return false;
}
bool StaticDataProfileInfoWrapperPass::doFinalization(Module &M) {
Info.reset();
return false;
}
INITIALIZE_PASS(StaticDataProfileInfoWrapperPass, "static-data-profile-info",
"Static Data Profile Info", false, true)
StaticDataProfileInfoWrapperPass::StaticDataProfileInfoWrapperPass()
: ImmutablePass(ID) {}
char StaticDataProfileInfoWrapperPass::ID = 0;
|
