1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
//===-- CSPreInliner.h - Profile guided preinliner ---------------- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_TOOLS_LLVM_PROFGEN_PGOINLINEADVISOR_H
#define LLVM_TOOLS_LLVM_PROFGEN_PGOINLINEADVISOR_H
#include "ProfiledBinary.h"
#include "llvm/ADT/PriorityQueue.h"
#include "llvm/ProfileData/ProfileCommon.h"
#include "llvm/ProfileData/SampleProf.h"
#include "llvm/Transforms/IPO/ProfiledCallGraph.h"
#include "llvm/Transforms/IPO/SampleContextTracker.h"
using namespace llvm;
using namespace sampleprof;
namespace llvm {
namespace sampleprof {
// Inline candidate seen from profile
struct ProfiledInlineCandidate {
ProfiledInlineCandidate(const FunctionSamples *Samples, uint64_t Count,
uint32_t Size)
: CalleeSamples(Samples), CallsiteCount(Count), SizeCost(Size) {}
// Context-sensitive function profile for inline candidate
const FunctionSamples *CalleeSamples;
// Call site count for an inline candidate
// TODO: make sure entry count for context profile and call site
// target count for corresponding call are consistent.
uint64_t CallsiteCount;
// Size proxy for function under particular call context.
uint64_t SizeCost;
};
// Inline candidate comparer using call site weight
struct ProfiledCandidateComparer {
bool operator()(const ProfiledInlineCandidate &LHS,
const ProfiledInlineCandidate &RHS) {
// Always prioritize inlining zero-sized functions as they do not affect the
// size budget. This could happen when all of the callee's code is gone and
// only pseudo probes are left.
if ((LHS.SizeCost == 0 || RHS.SizeCost == 0) &&
(LHS.SizeCost != RHS.SizeCost))
return RHS.SizeCost == 0;
if (LHS.CallsiteCount != RHS.CallsiteCount)
return LHS.CallsiteCount < RHS.CallsiteCount;
if (LHS.SizeCost != RHS.SizeCost)
return LHS.SizeCost > RHS.SizeCost;
// Tie breaker using GUID so we have stable/deterministic inlining order
assert(LHS.CalleeSamples && RHS.CalleeSamples &&
"Expect non-null FunctionSamples");
return LHS.CalleeSamples->getGUID(LHS.CalleeSamples->getName()) <
RHS.CalleeSamples->getGUID(RHS.CalleeSamples->getName());
}
};
using ProfiledCandidateQueue =
PriorityQueue<ProfiledInlineCandidate, std::vector<ProfiledInlineCandidate>,
ProfiledCandidateComparer>;
// Pre-compilation inliner based on context-sensitive profile.
// The PreInliner estimates inline decision using hotness from profile
// and cost estimation from machine code size. It helps merges context
// profile globally and achieves better post-inine profile quality, which
// otherwise won't be possible for ThinLTO. It also reduce context profile
// size by only keep context that is estimated to be inlined.
class CSPreInliner {
public:
CSPreInliner(SampleContextTracker &Tracker, ProfiledBinary &Binary,
ProfileSummary *Summary);
void run();
private:
bool getInlineCandidates(ProfiledCandidateQueue &CQueue,
const FunctionSamples *FCallerContextSamples);
std::vector<StringRef> buildTopDownOrder();
void processFunction(StringRef Name);
bool shouldInline(ProfiledInlineCandidate &Candidate);
uint32_t getFuncSize(const ContextTrieNode *ContextNode);
bool UseContextCost;
SampleContextTracker &ContextTracker;
ProfiledBinary &Binary;
ProfileSummary *Summary;
};
} // end namespace sampleprof
} // end namespace llvm
#endif
|
