1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
|
//===- SpillPlacement.cpp - Optimal Spill Code Placement ------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the spill code placement analysis.
//
// Each edge bundle corresponds to a node in a Hopfield network. Constraints on
// basic blocks are weighted by the block frequency and added to become the node
// bias.
//
// Transparent basic blocks have the variable live through, but don't care if it
// is spilled or in a register. These blocks become connections in the Hopfield
// network, again weighted by block frequency.
//
// The Hopfield network minimizes (possibly locally) its energy function:
//
// E = -sum_n V_n * ( B_n + sum_{n, m linked by b} V_m * F_b )
//
// The energy function represents the expected spill code execution frequency,
// or the cost of spilling. This is a Lyapunov function which never increases
// when a node is updated. It is guaranteed to converge to a local minimum.
//
//===----------------------------------------------------------------------===//
#include "SpillPlacement.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/CodeGen/EdgeBundles.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <utility>
using namespace llvm;
#define DEBUG_TYPE "spill-code-placement"
char SpillPlacement::ID = 0;
char &llvm::SpillPlacementID = SpillPlacement::ID;
INITIALIZE_PASS_BEGIN(SpillPlacement, DEBUG_TYPE,
"Spill Code Placement Analysis", true, true)
INITIALIZE_PASS_DEPENDENCY(EdgeBundles)
INITIALIZE_PASS_END(SpillPlacement, DEBUG_TYPE,
"Spill Code Placement Analysis", true, true)
void SpillPlacement::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
AU.addRequired<MachineBlockFrequencyInfo>();
AU.addRequiredTransitive<EdgeBundles>();
MachineFunctionPass::getAnalysisUsage(AU);
}
/// Node - Each edge bundle corresponds to a Hopfield node.
///
/// The node contains precomputed frequency data that only depends on the CFG,
/// but Bias and Links are computed each time placeSpills is called.
///
/// The node Value is positive when the variable should be in a register. The
/// value can change when linked nodes change, but convergence is very fast
/// because all weights are positive.
struct SpillPlacement::Node {
/// BiasN - Sum of blocks that prefer a spill.
BlockFrequency BiasN;
/// BiasP - Sum of blocks that prefer a register.
BlockFrequency BiasP;
/// Value - Output value of this node computed from the Bias and links.
/// This is always on of the values {-1, 0, 1}. A positive number means the
/// variable should go in a register through this bundle.
int Value;
using LinkVector = SmallVector<std::pair<BlockFrequency, unsigned>, 4>;
/// Links - (Weight, BundleNo) for all transparent blocks connecting to other
/// bundles. The weights are all positive block frequencies.
LinkVector Links;
/// SumLinkWeights - Cached sum of the weights of all links + ThresHold.
BlockFrequency SumLinkWeights;
/// preferReg - Return true when this node prefers to be in a register.
bool preferReg() const {
// Undecided nodes (Value==0) go on the stack.
return Value > 0;
}
/// mustSpill - Return True if this node is so biased that it must spill.
bool mustSpill() const {
// We must spill if Bias < -sum(weights) or the MustSpill flag was set.
// BiasN is saturated when MustSpill is set, make sure this still returns
// true when the RHS saturates. Note that SumLinkWeights includes Threshold.
return BiasN >= BiasP + SumLinkWeights;
}
/// clear - Reset per-query data, but preserve frequencies that only depend on
/// the CFG.
void clear(BlockFrequency Threshold) {
BiasN = BlockFrequency(0);
BiasP = BlockFrequency(0);
Value = 0;
SumLinkWeights = Threshold;
Links.clear();
}
/// addLink - Add a link to bundle b with weight w.
void addLink(unsigned b, BlockFrequency w) {
// Update cached sum.
SumLinkWeights += w;
// There can be multiple links to the same bundle, add them up.
for (std::pair<BlockFrequency, unsigned> &L : Links)
if (L.second == b) {
L.first += w;
return;
}
// This must be the first link to b.
Links.push_back(std::make_pair(w, b));
}
/// addBias - Bias this node.
void addBias(BlockFrequency freq, BorderConstraint direction) {
switch (direction) {
default:
break;
case PrefReg:
BiasP += freq;
break;
case PrefSpill:
BiasN += freq;
break;
case MustSpill:
BiasN = BlockFrequency::max();
break;
}
}
/// update - Recompute Value from Bias and Links. Return true when node
/// preference changes.
bool update(const Node nodes[], BlockFrequency Threshold) {
// Compute the weighted sum of inputs.
BlockFrequency SumN = BiasN;
BlockFrequency SumP = BiasP;
for (std::pair<BlockFrequency, unsigned> &L : Links) {
if (nodes[L.second].Value == -1)
SumN += L.first;
else if (nodes[L.second].Value == 1)
SumP += L.first;
}
// Each weighted sum is going to be less than the total frequency of the
// bundle. Ideally, we should simply set Value = sign(SumP - SumN), but we
// will add a dead zone around 0 for two reasons:
//
// 1. It avoids arbitrary bias when all links are 0 as is possible during
// initial iterations.
// 2. It helps tame rounding errors when the links nominally sum to 0.
//
bool Before = preferReg();
if (SumN >= SumP + Threshold)
Value = -1;
else if (SumP >= SumN + Threshold)
Value = 1;
else
Value = 0;
return Before != preferReg();
}
void getDissentingNeighbors(SparseSet<unsigned> &List,
const Node nodes[]) const {
for (const auto &Elt : Links) {
unsigned n = Elt.second;
// Neighbors that already have the same value are not going to
// change because of this node changing.
if (Value != nodes[n].Value)
List.insert(n);
}
}
};
bool SpillPlacement::runOnMachineFunction(MachineFunction &mf) {
MF = &mf;
bundles = &getAnalysis<EdgeBundles>();
assert(!nodes && "Leaking node array");
nodes = new Node[bundles->getNumBundles()];
TodoList.clear();
TodoList.setUniverse(bundles->getNumBundles());
// Compute total ingoing and outgoing block frequencies for all bundles.
BlockFrequencies.resize(mf.getNumBlockIDs());
MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
setThreshold(MBFI->getEntryFreq());
for (auto &I : mf) {
unsigned Num = I.getNumber();
BlockFrequencies[Num] = MBFI->getBlockFreq(&I);
}
// We never change the function.
return false;
}
void SpillPlacement::releaseMemory() {
delete[] nodes;
nodes = nullptr;
TodoList.clear();
}
/// activate - mark node n as active if it wasn't already.
void SpillPlacement::activate(unsigned n) {
TodoList.insert(n);
if (ActiveNodes->test(n))
return;
ActiveNodes->set(n);
nodes[n].clear(Threshold);
// Very large bundles usually come from big switches, indirect branches,
// landing pads, or loops with many 'continue' statements. It is difficult to
// allocate registers when so many different blocks are involved.
//
// Give a small negative bias to large bundles such that a substantial
// fraction of the connected blocks need to be interested before we consider
// expanding the region through the bundle. This helps compile time by
// limiting the number of blocks visited and the number of links in the
// Hopfield network.
if (bundles->getBlocks(n).size() > 100) {
nodes[n].BiasP = BlockFrequency(0);
BlockFrequency BiasN = MBFI->getEntryFreq();
BiasN >>= 4;
nodes[n].BiasN = BiasN;
}
}
/// Set the threshold for a given entry frequency.
///
/// Set the threshold relative to \c Entry. Since the threshold is used as a
/// bound on the open interval (-Threshold;Threshold), 1 is the minimum
/// threshold.
void SpillPlacement::setThreshold(BlockFrequency Entry) {
// Apparently 2 is a good threshold when Entry==2^14, but we need to scale
// it. Divide by 2^13, rounding as appropriate.
uint64_t Freq = Entry.getFrequency();
uint64_t Scaled = (Freq >> 13) + bool(Freq & (1 << 12));
Threshold = BlockFrequency(std::max(UINT64_C(1), Scaled));
}
/// addConstraints - Compute node biases and weights from a set of constraints.
/// Set a bit in NodeMask for each active node.
void SpillPlacement::addConstraints(ArrayRef<BlockConstraint> LiveBlocks) {
for (const BlockConstraint &LB : LiveBlocks) {
BlockFrequency Freq = BlockFrequencies[LB.Number];
// Live-in to block?
if (LB.Entry != DontCare) {
unsigned ib = bundles->getBundle(LB.Number, false);
activate(ib);
nodes[ib].addBias(Freq, LB.Entry);
}
// Live-out from block?
if (LB.Exit != DontCare) {
unsigned ob = bundles->getBundle(LB.Number, true);
activate(ob);
nodes[ob].addBias(Freq, LB.Exit);
}
}
}
/// addPrefSpill - Same as addConstraints(PrefSpill)
void SpillPlacement::addPrefSpill(ArrayRef<unsigned> Blocks, bool Strong) {
for (unsigned B : Blocks) {
BlockFrequency Freq = BlockFrequencies[B];
if (Strong)
Freq += Freq;
unsigned ib = bundles->getBundle(B, false);
unsigned ob = bundles->getBundle(B, true);
activate(ib);
activate(ob);
nodes[ib].addBias(Freq, PrefSpill);
nodes[ob].addBias(Freq, PrefSpill);
}
}
void SpillPlacement::addLinks(ArrayRef<unsigned> Links) {
for (unsigned Number : Links) {
unsigned ib = bundles->getBundle(Number, false);
unsigned ob = bundles->getBundle(Number, true);
// Ignore self-loops.
if (ib == ob)
continue;
activate(ib);
activate(ob);
BlockFrequency Freq = BlockFrequencies[Number];
nodes[ib].addLink(ob, Freq);
nodes[ob].addLink(ib, Freq);
}
}
bool SpillPlacement::scanActiveBundles() {
RecentPositive.clear();
for (unsigned n : ActiveNodes->set_bits()) {
update(n);
// A node that must spill, or a node without any links is not going to
// change its value ever again, so exclude it from iterations.
if (nodes[n].mustSpill())
continue;
if (nodes[n].preferReg())
RecentPositive.push_back(n);
}
return !RecentPositive.empty();
}
bool SpillPlacement::update(unsigned n) {
if (!nodes[n].update(nodes, Threshold))
return false;
nodes[n].getDissentingNeighbors(TodoList, nodes);
return true;
}
/// iterate - Repeatedly update the Hopfield nodes until stability or the
/// maximum number of iterations is reached.
void SpillPlacement::iterate() {
// We do not need to push those node in the todolist.
// They are already been proceeded as part of the previous iteration.
RecentPositive.clear();
// Since the last iteration, the todolist have been augmented by calls
// to addConstraints, addLinks, and co.
// Update the network energy starting at this new frontier.
// The call to ::update will add the nodes that changed into the todolist.
unsigned Limit = bundles->getNumBundles() * 10;
while(Limit-- > 0 && !TodoList.empty()) {
unsigned n = TodoList.pop_back_val();
if (!update(n))
continue;
if (nodes[n].preferReg())
RecentPositive.push_back(n);
}
}
void SpillPlacement::prepare(BitVector &RegBundles) {
RecentPositive.clear();
TodoList.clear();
// Reuse RegBundles as our ActiveNodes vector.
ActiveNodes = &RegBundles;
ActiveNodes->clear();
ActiveNodes->resize(bundles->getNumBundles());
}
bool
SpillPlacement::finish() {
assert(ActiveNodes && "Call prepare() first");
// Write preferences back to ActiveNodes.
bool Perfect = true;
for (unsigned n : ActiveNodes->set_bits())
if (!nodes[n].preferReg()) {
ActiveNodes->reset(n);
Perfect = false;
}
ActiveNodes = nullptr;
return Perfect;
}
void SpillPlacement::BlockConstraint::print(raw_ostream &OS) const {
auto toString = [](BorderConstraint C) -> StringRef {
switch(C) {
case DontCare: return "DontCare";
case PrefReg: return "PrefReg";
case PrefSpill: return "PrefSpill";
case PrefBoth: return "PrefBoth";
case MustSpill: return "MustSpill";
};
llvm_unreachable("uncovered switch");
};
dbgs() << "{" << Number << ", "
<< toString(Entry) << ", "
<< toString(Exit) << ", "
<< (ChangesValue ? "changes" : "no change") << "}";
}
void SpillPlacement::BlockConstraint::dump() const {
print(dbgs());
dbgs() << "\n";
}
|