1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
//===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements PseudoProbeInserter pass, which inserts pseudo probe
// annotations for call instructions with a pseudo-probe-specific dwarf
// discriminator. such discriminator indicates that the call instruction comes
// with a pseudo probe, and the discriminator value holds information to
// identify the corresponding counter.
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PseudoProbe.h"
#include "llvm/InitializePasses.h"
#define DEBUG_TYPE "pseudo-probe-inserter"
using namespace llvm;
namespace {
class PseudoProbeInserter : public MachineFunctionPass {
public:
static char ID;
PseudoProbeInserter() : MachineFunctionPass(ID) {
initializePseudoProbeInserterPass(*PassRegistry::getPassRegistry());
}
StringRef getPassName() const override { return "Pseudo Probe Inserter"; }
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
}
bool doInitialization(Module &M) override {
ShouldRun = M.getNamedMetadata(PseudoProbeDescMetadataName);
return false;
}
bool runOnMachineFunction(MachineFunction &MF) override {
if (!ShouldRun)
return false;
const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
bool Changed = false;
for (MachineBasicBlock &MBB : MF) {
MachineInstr *FirstInstr = nullptr;
for (MachineInstr &MI : MBB) {
if (!MI.isPseudo())
FirstInstr = &MI;
if (MI.isCall()) {
if (DILocation *DL = MI.getDebugLoc()) {
auto Value = DL->getDiscriminator();
if (DILocation::isPseudoProbeDiscriminator(Value)) {
BuildMI(MBB, MI, DL, TII->get(TargetOpcode::PSEUDO_PROBE))
.addImm(getFuncGUID(MF.getFunction().getParent(), DL))
.addImm(
PseudoProbeDwarfDiscriminator::extractProbeIndex(Value))
.addImm(
PseudoProbeDwarfDiscriminator::extractProbeType(Value))
.addImm(PseudoProbeDwarfDiscriminator::extractProbeAttributes(
Value));
Changed = true;
}
}
}
}
// Walk the block backwards, move PSEUDO_PROBE before the first real
// instruction to fix out-of-order probes. There is a problem with probes
// as the terminator of the block. During the offline counts processing,
// the samples collected on the first physical instruction following a
// probe will be counted towards the probe. This logically equals to
// treating the instruction next to a probe as if it is from the same
// block of the probe. This is accurate most of the time unless the
// instruction can be reached from multiple flows, which means it actually
// starts a new block. Samples collected on such probes may cause
// imprecision with the counts inference algorithm. Fortunately, if
// there are still other native instructions preceding the probe we can
// use them as a place holder to collect samples for the probe.
if (FirstInstr) {
auto MII = MBB.rbegin();
while (MII != MBB.rend()) {
// Skip all pseudo probes followed by a real instruction since they
// are not dangling.
if (!MII->isPseudo())
break;
auto Cur = MII++;
if (Cur->getOpcode() != TargetOpcode::PSEUDO_PROBE)
continue;
// Move the dangling probe before FirstInstr.
auto *ProbeInstr = &*Cur;
MBB.remove(ProbeInstr);
MBB.insert(FirstInstr, ProbeInstr);
Changed = true;
}
} else {
// Probes not surrounded by any real instructions in the same block are
// called dangling probes. Since there's no good way to pick up a sample
// collection point for dangling probes at compile time, they are being
// removed so that the profile correlation tool will not report any
// samples collected for them and it's up to the counts inference tool
// to get them a reasonable count.
SmallVector<MachineInstr *, 4> ToBeRemoved;
for (MachineInstr &MI : MBB) {
if (MI.isPseudoProbe())
ToBeRemoved.push_back(&MI);
}
for (auto *MI : ToBeRemoved)
MI->eraseFromParent();
Changed |= !ToBeRemoved.empty();
}
}
return Changed;
}
private:
uint64_t getFuncGUID(Module *M, DILocation *DL) {
auto Name = DL->getSubprogramLinkageName();
return Function::getGUID(Name);
}
bool ShouldRun = false;
};
} // namespace
char PseudoProbeInserter::ID = 0;
INITIALIZE_PASS_BEGIN(PseudoProbeInserter, DEBUG_TYPE,
"Insert pseudo probe annotations for value profiling",
false, false)
INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
INITIALIZE_PASS_END(PseudoProbeInserter, DEBUG_TYPE,
"Insert pseudo probe annotations for value profiling",
false, false)
FunctionPass *llvm::createPseudoProbeInserter() {
return new PseudoProbeInserter();
}
|
