1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
|
//===-- Target.cpp ----------------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "../Target.h"
#include "AArch64.h"
#include "AArch64RegisterInfo.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#if defined(__aarch64__) && defined(__linux__)
#include <sys/prctl.h> // For PR_PAC_* constants
#ifndef PR_PAC_APIAKEY
#define PR_PAC_APIAKEY (1UL << 0)
#endif
#ifndef PR_PAC_APIBKEY
#define PR_PAC_APIBKEY (1UL << 1)
#endif
#ifndef PR_PAC_APDAKEY
#define PR_PAC_APDAKEY (1UL << 2)
#endif
#ifndef PR_PAC_APDBKEY
#define PR_PAC_APDBKEY (1UL << 3)
#endif
#endif
#define GET_AVAILABLE_OPCODE_CHECKER
#include "AArch64GenInstrInfo.inc"
namespace llvm {
namespace exegesis {
static unsigned getLoadImmediateOpcode(unsigned RegBitWidth) {
switch (RegBitWidth) {
case 32:
return AArch64::MOVi32imm;
case 64:
return AArch64::MOVi64imm;
}
llvm_unreachable("Invalid Value Width");
}
// Generates instruction to load an immediate value into a register.
static MCInst loadImmediate(MCRegister Reg, unsigned RegBitWidth,
const APInt &Value) {
assert(Value.getBitWidth() <= RegBitWidth &&
"Value must fit in the Register");
return MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
.addReg(Reg)
.addImm(Value.getZExtValue());
}
static MCInst loadZPRImmediate(MCRegister Reg, unsigned RegBitWidth,
const APInt &Value) {
assert(Value.getZExtValue() < (1 << 7) &&
"Value must be in the range of the immediate opcode");
return MCInstBuilder(AArch64::DUP_ZI_D)
.addReg(Reg)
.addImm(Value.getZExtValue())
.addImm(0);
}
static MCInst loadPPRImmediate(MCRegister Reg, unsigned RegBitWidth,
const APInt &Value) {
// For PPR, we typically use PTRUE instruction to set predicate registers
return MCInstBuilder(AArch64::PTRUE_B)
.addReg(Reg)
.addImm(31); // All lanes true for 16 bits
}
// Generates instructions to load an immediate value into an FPCR register.
static std::vector<MCInst>
loadFPCRImmediate(MCRegister Reg, unsigned RegBitWidth, const APInt &Value) {
MCRegister TempReg = AArch64::X8;
MCInst LoadImm = MCInstBuilder(AArch64::MOVi64imm).addReg(TempReg).addImm(0);
MCInst MoveToFPCR =
MCInstBuilder(AArch64::MSR).addImm(AArch64SysReg::FPCR).addReg(TempReg);
return {LoadImm, MoveToFPCR};
}
// Fetch base-instruction to load an FP immediate value into a register.
static unsigned getLoadFPImmediateOpcode(unsigned RegBitWidth) {
switch (RegBitWidth) {
case 16:
return AArch64::FMOVH0; // FMOVHi;
case 32:
return AArch64::FMOVS0; // FMOVSi;
case 64:
return AArch64::MOVID; // FMOVDi;
case 128:
return AArch64::MOVIv2d_ns;
}
llvm_unreachable("Invalid Value Width");
}
// Generates instruction to load an FP immediate value into a register.
static MCInst loadFPImmediate(MCRegister Reg, unsigned RegBitWidth,
const APInt &Value) {
assert(Value.getZExtValue() == 0 && "Expected initialisation value 0");
MCInst Instructions =
MCInstBuilder(getLoadFPImmediateOpcode(RegBitWidth)).addReg(Reg);
if (RegBitWidth >= 64)
Instructions.addOperand(MCOperand::createImm(Value.getZExtValue()));
return Instructions;
}
#include "AArch64GenExegesis.inc"
namespace {
// Use X19 as the loop counter register since it's a callee-saved register
// that's available for temporary use.
constexpr const MCPhysReg kDefaultLoopCounterReg = AArch64::X19;
class ExegesisAArch64Target : public ExegesisTarget {
public:
ExegesisAArch64Target()
: ExegesisTarget(AArch64CpuPfmCounters, AArch64_MC::isOpcodeAvailable) {}
Error randomizeTargetMCOperand(const Instruction &Instr, const Variable &Var,
MCOperand &AssignedValue,
const BitVector &ForbiddenRegs) const override;
private:
std::vector<MCInst> setRegTo(const MCSubtargetInfo &STI, MCRegister Reg,
const APInt &Value) const override {
if (AArch64::GPR32RegClass.contains(Reg))
return {loadImmediate(Reg, 32, Value)};
if (AArch64::GPR64RegClass.contains(Reg))
return {loadImmediate(Reg, 64, Value)};
if (AArch64::PPRRegClass.contains(Reg))
return {loadPPRImmediate(Reg, 16, Value)};
if (AArch64::FPR8RegClass.contains(Reg))
return {loadFPImmediate(Reg - AArch64::B0 + AArch64::D0, 64, Value)};
if (AArch64::FPR16RegClass.contains(Reg))
return {loadFPImmediate(Reg, 16, Value)};
if (AArch64::FPR32RegClass.contains(Reg))
return {loadFPImmediate(Reg, 32, Value)};
if (AArch64::FPR64RegClass.contains(Reg))
return {loadFPImmediate(Reg, 64, Value)};
if (AArch64::FPR128RegClass.contains(Reg))
return {loadFPImmediate(Reg, 128, Value)};
if (AArch64::ZPRRegClass.contains(Reg))
return {loadZPRImmediate(Reg, 128, Value)};
if (Reg == AArch64::FPCR)
return {loadFPCRImmediate(Reg, 32, Value)};
errs() << "setRegTo is not implemented, results will be unreliable\n";
return {};
}
MCRegister getDefaultLoopCounterRegister(const Triple &) const override {
return kDefaultLoopCounterReg;
}
void decrementLoopCounterAndJump(MachineBasicBlock &MBB,
MachineBasicBlock &TargetMBB,
const MCInstrInfo &MII,
MCRegister LoopRegister) const override {
// subs LoopRegister, LoopRegister, #1
BuildMI(&MBB, DebugLoc(), MII.get(AArch64::SUBSXri))
.addDef(LoopRegister)
.addUse(LoopRegister)
.addImm(1) // Subtract 1
.addImm(0); // No shift amount
// b.ne TargetMBB
BuildMI(&MBB, DebugLoc(), MII.get(AArch64::Bcc))
.addImm(AArch64CC::NE)
.addMBB(&TargetMBB);
}
// Registers that should not be selected for use in snippets.
const MCPhysReg UnavailableRegisters[1] = {kDefaultLoopCounterReg};
ArrayRef<MCPhysReg> getUnavailableRegisters() const override {
return UnavailableRegisters;
}
bool matchesArch(Triple::ArchType Arch) const override {
return Arch == Triple::aarch64 || Arch == Triple::aarch64_be;
}
void addTargetSpecificPasses(PassManagerBase &PM) const override {
// Function return is a pseudo-instruction that needs to be expanded
PM.add(createAArch64ExpandPseudoPass());
}
};
Error ExegesisAArch64Target::randomizeTargetMCOperand(
const Instruction &Instr, const Variable &Var, MCOperand &AssignedValue,
const BitVector &ForbiddenRegs) const {
const Operand &Op = Instr.getPrimaryOperand(Var);
const auto OperandType = Op.getExplicitOperandInfo().OperandType;
// NOTE: To resolve "Not all operands were initialized by snippet generator"
// Requires OperandType to be defined for such opcode's operands in AArch64
// tablegen files. And omit introduced OperandType(s).
// Hacky Fix: Defaulting all OPERAND_UNKNOWN to immediate value 0 works with a
// limitation that it introduces illegal instruction error for system
// instructions. System instructions will need to be omitted with OperandType
// or opcode specific values to avoid generating invalid encodings or
// unreliable benchmark results for these system-level instructions.
// Implement opcode-specific immediate value handling for system instrs:
// - MRS/MSR: Use valid system register encodings (e.g., NZCV, FPCR, FPSR)
// - MSRpstatesvcrImm1: Use valid PSTATE field encodings (e.g., SPSel,
// DAIFSet)
// - SYSLxt/SYSxt: Use valid system instruction encodings with proper
// CRn/CRm/op values
// - UDF: Use valid undefined instruction immediate ranges (0-65535)
switch (OperandType) {
// MSL (Masking Shift Left) imm operand for 32-bit splatted SIMD constants
// Correspond to AArch64InstructionSelector::tryAdvSIMDModImm321s()
case llvm::AArch64::OPERAND_SHIFT_MSL: {
// There are two valid encodings:
// - Type 7: imm at [15:8], [47:40], shift = 264 (0x108) → msl #8
// - Type 8: imm at [23:16], [55:48], shift = 272 (0x110) → msl #16
// Corresponds AArch64_AM::encodeAdvSIMDModImmType7()
// But, v2s_msl and v4s_msl instructions accept either form,
// Thus, Arbitrarily chosing 264 (msl #8) for simplicity.
AssignedValue = MCOperand::createImm(264);
return Error::success();
}
case llvm::AArch64::OPERAND_IMPLICIT_IMM_0:
AssignedValue = MCOperand::createImm(0);
return Error::success();
case MCOI::OperandType::OPERAND_PCREL:
AssignedValue = MCOperand::createImm(8);
return Error::success();
default:
break;
}
return make_error<Failure>(
Twine("Unimplemented operand type: MCOI::OperandType:")
.concat(Twine(static_cast<int>(OperandType))));
}
} // namespace
static ExegesisTarget *getTheExegesisAArch64Target() {
static ExegesisAArch64Target Target;
return &Target;
}
void InitializeAArch64ExegesisTarget() {
ExegesisTarget::registerTarget(getTheExegesisAArch64Target());
}
} // namespace exegesis
} // namespace llvm
|
