aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/CGData/CodeGenDataWriter.cpp4
-rw-r--r--llvm/lib/CodeGen/RDFLiveness.cpp9
-rw-r--r--llvm/lib/MC/DXContainerRootSignature.cpp5
-rw-r--r--llvm/lib/Object/ArchiveWriter.cpp20
-rw-r--r--llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp12
-rw-r--r--llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp2
-rw-r--r--llvm/lib/ProfileData/InstrProf.cpp2
-rw-r--r--llvm/lib/ProfileData/InstrProfReader.cpp10
-rw-r--r--llvm/lib/Target/Hexagon/RDFCopy.cpp2
-rw-r--r--llvm/lib/Target/Hexagon/RDFCopy.h8
-rw-r--r--llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp51
-rw-r--r--llvm/lib/Target/SPIRV/SPIRVBuiltins.td24
-rw-r--r--llvm/lib/Target/SPIRV/SPIRVInstrInfo.td33
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlan.cpp15
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp17
15 files changed, 172 insertions, 42 deletions
diff --git a/llvm/lib/CGData/CodeGenDataWriter.cpp b/llvm/lib/CGData/CodeGenDataWriter.cpp
index 14a8558..a2bbcee 100644
--- a/llvm/lib/CGData/CodeGenDataWriter.cpp
+++ b/llvm/lib/CGData/CodeGenDataWriter.cpp
@@ -40,7 +40,7 @@ void CGDataOStream::patch(ArrayRef<CGDataPatchItem> P) {
for (const auto &K : P) {
for (size_t I = 0; I < K.D.size(); ++I) {
uint64_t Bytes =
- endian::byte_swap<uint64_t, llvm::endianness::little>(K.D[I]);
+ endian::byte_swap<uint64_t>(K.D[I], llvm::endianness::little);
Data.replace(K.Pos + I * sizeof(uint64_t), sizeof(uint64_t),
reinterpret_cast<const char *>(&Bytes), sizeof(uint64_t));
}
@@ -52,7 +52,7 @@ void CGDataOStream::patch(ArrayRef<CGDataPatchItem> P) {
for (const auto &K : P) {
for (size_t I = 0; I < K.D.size(); ++I) {
uint64_t Bytes =
- endian::byte_swap<uint64_t, llvm::endianness::little>(K.D[I]);
+ endian::byte_swap<uint64_t>(K.D[I], llvm::endianness::little);
VOStream.pwrite(reinterpret_cast<const char *>(&Bytes),
sizeof(uint64_t), K.Pos + I * sizeof(uint64_t));
}
diff --git a/llvm/lib/CodeGen/RDFLiveness.cpp b/llvm/lib/CodeGen/RDFLiveness.cpp
index 318422b..2e1cf49 100644
--- a/llvm/lib/CodeGen/RDFLiveness.cpp
+++ b/llvm/lib/CodeGen/RDFLiveness.cpp
@@ -652,8 +652,9 @@ void Liveness::computePhiInfo() {
// defs, cache the result of subtracting these defs from a given register
// ref.
using RefHash = std::hash<RegisterRef>;
- using RefEqual = std::equal_to<RegisterRef>;
- using SubMap = std::unordered_map<RegisterRef, RegisterRef>;
+ using RefEqual = RegisterRefEqualTo;
+ using SubMap =
+ std::unordered_map<RegisterRef, RegisterRef, RefHash, RefEqual>;
std::unordered_map<RegisterAggr, SubMap> Subs;
auto ClearIn = [](RegisterRef RR, const RegisterAggr &Mid, SubMap &SM) {
if (Mid.empty())
@@ -868,7 +869,7 @@ void Liveness::computeLiveIns() {
std::vector<RegisterRef> LV;
for (const MachineBasicBlock::RegisterMaskPair &LI : B.liveins())
LV.push_back(RegisterRef(LI.PhysReg, LI.LaneMask));
- llvm::sort(LV, std::less<RegisterRef>(PRI));
+ llvm::sort(LV, RegisterRefLess(PRI));
dbgs() << printMBBReference(B) << "\t rec = {";
for (auto I : LV)
dbgs() << ' ' << Print(I, DFG);
@@ -878,7 +879,7 @@ void Liveness::computeLiveIns() {
LV.clear();
for (RegisterRef RR : LiveMap[&B].refs())
LV.push_back(RR);
- llvm::sort(LV, std::less<RegisterRef>(PRI));
+ llvm::sort(LV, RegisterRefLess(PRI));
dbgs() << "\tcomp = {";
for (auto I : LV)
dbgs() << ' ' << Print(I, DFG);
diff --git a/llvm/lib/MC/DXContainerRootSignature.cpp b/llvm/lib/MC/DXContainerRootSignature.cpp
index 2338370..713aa3d8 100644
--- a/llvm/lib/MC/DXContainerRootSignature.cpp
+++ b/llvm/lib/MC/DXContainerRootSignature.cpp
@@ -23,9 +23,8 @@ static uint32_t writePlaceholder(raw_svector_ostream &Stream) {
static uint32_t rewriteOffsetToCurrentByte(raw_svector_ostream &Stream,
uint32_t Offset) {
uint32_t ByteOffset = Stream.tell();
- uint32_t Value =
- support::endian::byte_swap<uint32_t, llvm::endianness::little>(
- ByteOffset);
+ uint32_t Value = support::endian::byte_swap<uint32_t>(
+ ByteOffset, llvm::endianness::little);
Stream.pwrite(reinterpret_cast<const char *>(&Value), sizeof(Value), Offset);
return ByteOffset;
}
diff --git a/llvm/lib/Object/ArchiveWriter.cpp b/llvm/lib/Object/ArchiveWriter.cpp
index 6fc0889..a112597 100644
--- a/llvm/lib/Object/ArchiveWriter.cpp
+++ b/llvm/lib/Object/ArchiveWriter.cpp
@@ -1119,10 +1119,26 @@ Error writeArchiveToStream(raw_ostream &Out,
// to switch to 64-bit. Note that the file can be larger than 4GB as long as
// the last member starts before the 4GB offset.
if (*HeadersSize + LastMemberHeaderOffset >= Sym64Threshold) {
- if (Kind == object::Archive::K_DARWIN)
+ switch (Kind) {
+ case object::Archive::K_COFF:
+ // COFF format has no 64-bit version, so we use GNU64 instead.
+ if (!SymMap.Map.empty() && !SymMap.ECMap.empty())
+ // Only the COFF format supports the ECSYMBOLS section, so don’t use
+ // GNU64 when two symbol maps are required.
+ return make_error<object::GenericBinaryError>(
+ "Archive is too large: ARM64X does not support archives larger "
+ "than 4GB");
+ // Since this changes the headers, we need to recalculate everything.
+ return writeArchiveToStream(Out, NewMembers, WriteSymtab,
+ object::Archive::K_GNU64, Deterministic,
+ Thin, IsEC, Warn);
+ case object::Archive::K_DARWIN:
Kind = object::Archive::K_DARWIN64;
- else
+ break;
+ default:
Kind = object::Archive::K_GNU64;
+ break;
+ }
HeadersSize.reset();
}
}
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
index fc2577e..075ad8d 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
@@ -949,9 +949,9 @@ loadTestingFormat(StringRef Data, StringRef CompilationDir) {
if (Data.size() < sizeof(uint64_t))
return make_error<CoverageMapError>(coveragemap_error::malformed,
"the size of data is too small");
- auto TestingVersion =
- support::endian::byte_swap<uint64_t, llvm::endianness::little>(
- *reinterpret_cast<const uint64_t *>(Data.data()));
+ auto TestingVersion = support::endian::byte_swap<uint64_t>(
+ *reinterpret_cast<const uint64_t *>(Data.data()),
+ llvm::endianness::little);
Data = Data.substr(sizeof(uint64_t));
// Read the ProfileNames data.
@@ -1274,9 +1274,9 @@ BinaryCoverageReader::create(
std::vector<std::unique_ptr<BinaryCoverageReader>> Readers;
if (ObjectBuffer.getBuffer().size() > sizeof(TestingFormatMagic)) {
- uint64_t Magic =
- support::endian::byte_swap<uint64_t, llvm::endianness::little>(
- *reinterpret_cast<const uint64_t *>(ObjectBuffer.getBufferStart()));
+ uint64_t Magic = support::endian::byte_swap<uint64_t>(
+ *reinterpret_cast<const uint64_t *>(ObjectBuffer.getBufferStart()),
+ llvm::endianness::little);
if (Magic == TestingFormatMagic) {
// This is a special format used for testing.
auto ReaderOrErr =
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
index 12b1687..3875f01 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
@@ -292,7 +292,7 @@ void CoverageMappingWriter::write(raw_ostream &OS) {
void TestingFormatWriter::write(raw_ostream &OS, TestingFormatVersion Version) {
auto ByteSwap = [](uint64_t N) {
- return support::endian::byte_swap<uint64_t, llvm::endianness::little>(N);
+ return support::endian::byte_swap<uint64_t>(N, llvm::endianness::little);
};
// Output a 64bit magic number.
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index e1c6315..3c8e44a 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -292,7 +292,7 @@ void ProfOStream::patch(ArrayRef<PatchItem> P) {
for (const auto &K : P) {
for (int I = 0, E = K.D.size(); I != E; I++) {
uint64_t Bytes =
- endian::byte_swap<uint64_t, llvm::endianness::little>(K.D[I]);
+ endian::byte_swap<uint64_t>(K.D[I], llvm::endianness::little);
Data.replace(K.Pos + I * sizeof(uint64_t), sizeof(uint64_t),
(const char *)&Bytes, sizeof(uint64_t));
}
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 1da92ea..d2ae4b5 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -1186,10 +1186,10 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
if (Version >= IndexedInstrProf::Version4) {
const IndexedInstrProf::Summary *SummaryInLE =
reinterpret_cast<const IndexedInstrProf::Summary *>(Cur);
- uint64_t NFields = endian::byte_swap<uint64_t, llvm::endianness::little>(
- SummaryInLE->NumSummaryFields);
- uint64_t NEntries = endian::byte_swap<uint64_t, llvm::endianness::little>(
- SummaryInLE->NumCutoffEntries);
+ uint64_t NFields = endian::byte_swap<uint64_t>(
+ SummaryInLE->NumSummaryFields, llvm::endianness::little);
+ uint64_t NEntries = endian::byte_swap<uint64_t>(
+ SummaryInLE->NumCutoffEntries, llvm::endianness::little);
uint32_t SummarySize =
IndexedInstrProf::Summary::getSize(NFields, NEntries);
std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
@@ -1198,7 +1198,7 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
const uint64_t *Src = reinterpret_cast<const uint64_t *>(SummaryInLE);
uint64_t *Dst = reinterpret_cast<uint64_t *>(SummaryData.get());
for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
- Dst[I] = endian::byte_swap<uint64_t, llvm::endianness::little>(Src[I]);
+ Dst[I] = endian::byte_swap<uint64_t>(Src[I], llvm::endianness::little);
SummaryEntryVector DetailedSummary;
for (unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) {
diff --git a/llvm/lib/Target/Hexagon/RDFCopy.cpp b/llvm/lib/Target/Hexagon/RDFCopy.cpp
index fafdad0..3b1d3bd 100644
--- a/llvm/lib/Target/Hexagon/RDFCopy.cpp
+++ b/llvm/lib/Target/Hexagon/RDFCopy.cpp
@@ -108,7 +108,7 @@ bool CopyPropagation::scanBlock(MachineBasicBlock *B) {
for (NodeAddr<InstrNode*> IA : BA.Addr->members(DFG)) {
if (DFG.IsCode<NodeAttrs::Stmt>(IA)) {
NodeAddr<StmtNode*> SA = IA;
- EqualityMap EM(std::less<RegisterRef>(DFG.getPRI()));
+ EqualityMap EM(RegisterRefLess(DFG.getPRI()));
if (interpretAsCopy(SA.Addr->getCode(), EM))
recordCopy(SA, EM);
}
diff --git a/llvm/lib/Target/Hexagon/RDFCopy.h b/llvm/lib/Target/Hexagon/RDFCopy.h
index e4fb898..92b2c65 100644
--- a/llvm/lib/Target/Hexagon/RDFCopy.h
+++ b/llvm/lib/Target/Hexagon/RDFCopy.h
@@ -25,8 +25,8 @@ class MachineInstr;
namespace rdf {
struct CopyPropagation {
- CopyPropagation(DataFlowGraph &dfg) : MDT(dfg.getDT()), DFG(dfg),
- RDefMap(std::less<RegisterRef>(DFG.getPRI())) {}
+ CopyPropagation(DataFlowGraph &dfg)
+ : MDT(dfg.getDT()), DFG(dfg), RDefMap(RegisterRefLess(DFG.getPRI())) {}
virtual ~CopyPropagation() = default;
@@ -35,7 +35,7 @@ namespace rdf {
bool trace() const { return Trace; }
DataFlowGraph &getDFG() { return DFG; }
- using EqualityMap = std::map<RegisterRef, RegisterRef>;
+ using EqualityMap = std::map<RegisterRef, RegisterRef, RegisterRefLess>;
virtual bool interpretAsCopy(const MachineInstr *MI, EqualityMap &EM);
private:
@@ -45,7 +45,7 @@ namespace rdf {
bool Trace = false;
// map: register -> (map: stmt -> reaching def)
- std::map<RegisterRef,std::map<NodeId,NodeId>> RDefMap;
+ std::map<RegisterRef, std::map<NodeId, NodeId>, RegisterRefLess> RDefMap;
// map: statement -> (map: dst reg -> src reg)
std::map<NodeId, EqualityMap> CopyMap;
std::vector<NodeId> Copies;
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 86f4459..f704d3a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -1096,6 +1096,41 @@ static bool build2DBlockIOINTELInst(const SPIRV::IncomingCall *Call,
return true;
}
+static bool buildPipeInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
+ unsigned Scope, MachineIRBuilder &MIRBuilder,
+ SPIRVGlobalRegistry *GR) {
+ switch (Opcode) {
+ case SPIRV::OpCommitReadPipe:
+ case SPIRV::OpCommitWritePipe:
+ return buildOpFromWrapper(MIRBuilder, Opcode, Call, Register(0));
+ case SPIRV::OpGroupCommitReadPipe:
+ case SPIRV::OpGroupCommitWritePipe:
+ case SPIRV::OpGroupReserveReadPipePackets:
+ case SPIRV::OpGroupReserveWritePipePackets: {
+ Register ScopeConstReg =
+ MIRBuilder.buildConstant(LLT::scalar(32), Scope).getReg(0);
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+ MRI->setRegClass(ScopeConstReg, &SPIRV::iIDRegClass);
+ MachineInstrBuilder MIB;
+ MIB = MIRBuilder.buildInstr(Opcode);
+ // Add Return register and type.
+ if (Opcode == SPIRV::OpGroupReserveReadPipePackets ||
+ Opcode == SPIRV::OpGroupReserveWritePipePackets)
+ MIB.addDef(Call->ReturnRegister)
+ .addUse(GR->getSPIRVTypeID(Call->ReturnType));
+
+ MIB.addUse(ScopeConstReg);
+ for (unsigned int i = 0; i < Call->Arguments.size(); ++i)
+ MIB.addUse(Call->Arguments[i]);
+
+ return true;
+ }
+ default:
+ return buildOpFromWrapper(MIRBuilder, Opcode, Call,
+ GR->getSPIRVTypeID(Call->ReturnType));
+ }
+}
+
static unsigned getNumComponentsForDim(SPIRV::Dim::Dim dim) {
switch (dim) {
case SPIRV::Dim::DIM_1D:
@@ -2350,6 +2385,20 @@ static bool generate2DBlockIOINTELInst(const SPIRV::IncomingCall *Call,
return build2DBlockIOINTELInst(Call, Opcode, MIRBuilder, GR);
}
+static bool generatePipeInst(const SPIRV::IncomingCall *Call,
+ MachineIRBuilder &MIRBuilder,
+ SPIRVGlobalRegistry *GR) {
+ const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
+ unsigned Opcode =
+ SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;
+
+ unsigned Scope = SPIRV::Scope::Workgroup;
+ if (Builtin->Name.contains("sub_group"))
+ Scope = SPIRV::Scope::Subgroup;
+
+ return buildPipeInst(Call, Opcode, Scope, MIRBuilder, GR);
+}
+
static bool buildNDRange(const SPIRV::IncomingCall *Call,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
@@ -2948,6 +2997,8 @@ std::optional<bool> lowerBuiltin(const StringRef DemangledCall,
return generateTernaryBitwiseFunctionINTELInst(Call.get(), MIRBuilder, GR);
case SPIRV::Block2DLoadStore:
return generate2DBlockIOINTELInst(Call.get(), MIRBuilder, GR);
+ case SPIRV::Pipe:
+ return generatePipeInst(Call.get(), MIRBuilder, GR);
}
return false;
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index d08560b..2a8deb6 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -69,6 +69,7 @@ def ExtendedBitOps : BuiltinGroup;
def BindlessINTEL : BuiltinGroup;
def TernaryBitwiseINTEL : BuiltinGroup;
def Block2DLoadStore : BuiltinGroup;
+def Pipe : BuiltinGroup;
//===----------------------------------------------------------------------===//
// Class defining a demangled builtin record. The information in the record
@@ -633,6 +634,29 @@ defm : DemangledNativeBuiltin<"__spirv_AtomicSMax", OpenCL_std, Atomic, 4, 4, Op
defm : DemangledNativeBuiltin<"__spirv_AtomicUMin", OpenCL_std, Atomic, 4, 4, OpAtomicUMin>;
defm : DemangledNativeBuiltin<"__spirv_AtomicUMax", OpenCL_std, Atomic, 4, 4, OpAtomicUMax>;
+// Pipe Instruction
+defm : DemangledNativeBuiltin<"__read_pipe_2", OpenCL_std, Pipe,2, 2, OpReadPipe>;
+defm : DemangledNativeBuiltin<"__write_pipe_2", OpenCL_std, Pipe, 2, 2, OpWritePipe>;
+defm : DemangledNativeBuiltin<"__read_pipe_4", OpenCL_std, Pipe,4, 4, OpReservedReadPipe>;
+defm : DemangledNativeBuiltin<"__write_pipe_4", OpenCL_std, Pipe, 4, 4, OpReservedWritePipe>;
+defm : DemangledNativeBuiltin<"__reserve_read_pipe", OpenCL_std, Pipe, 2, 2, OpReserveReadPipePackets>;
+defm : DemangledNativeBuiltin<"__reserve_write_pipe", OpenCL_std, Pipe, 2, 2, OpReserveWritePipePackets>;
+defm : DemangledNativeBuiltin<"__commit_read_pipe", OpenCL_std, Pipe, 2, 2, OpCommitReadPipe>;
+defm : DemangledNativeBuiltin<"__commit_write_pipe", OpenCL_std, Pipe, 2, 2, OpCommitWritePipe>;
+defm : DemangledNativeBuiltin<"is_valid_reserve_id", OpenCL_std, Pipe, 1, 1, OpIsValidReserveId>;
+defm : DemangledNativeBuiltin<"__get_pipe_num_packets_ro", OpenCL_std, Pipe, 1, 1, OpGetNumPipePackets>;
+defm : DemangledNativeBuiltin<"__get_pipe_max_packets_ro", OpenCL_std, Pipe, 1, 1, OpGetMaxPipePackets>;
+defm : DemangledNativeBuiltin<"__get_pipe_num_packets_wo", OpenCL_std, Pipe, 1, 1, OpGetNumPipePackets>;
+defm : DemangledNativeBuiltin<"__get_pipe_max_packets_wo", OpenCL_std, Pipe, 1, 1, OpGetMaxPipePackets>;
+defm : DemangledNativeBuiltin<"__work_group_reserve_read_pipe", OpenCL_std, Pipe, 2, 2, OpGroupReserveReadPipePackets>;
+defm : DemangledNativeBuiltin<"__work_group_reserve_write_pipe", OpenCL_std, Pipe, 2, 2, OpGroupReserveWritePipePackets>;
+defm : DemangledNativeBuiltin<"__work_group_commit_read_pipe", OpenCL_std, Pipe, 2, 2, OpGroupCommitReadPipe>;
+defm : DemangledNativeBuiltin<"__work_group_commit_write_pipe", OpenCL_std, Pipe, 2, 2, OpGroupCommitWritePipe>;
+defm : DemangledNativeBuiltin<"__sub_group_reserve_read_pipe", OpenCL_std, Pipe, 2, 2, OpGroupReserveReadPipePackets>;
+defm : DemangledNativeBuiltin<"__sub_group_reserve_write_pipe", OpenCL_std, Pipe, 2, 2, OpGroupReserveWritePipePackets>;
+defm : DemangledNativeBuiltin<"__sub_group_commit_read_pipe", OpenCL_std, Pipe, 2, 2, OpGroupCommitReadPipe>;
+defm : DemangledNativeBuiltin<"__sub_group_commit_write_pipe", OpenCL_std, Pipe, 2, 2, OpGroupCommitWritePipe>;
+
// Barrier builtin records:
defm : DemangledNativeBuiltin<"barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>;
defm : DemangledNativeBuiltin<"work_group_barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index 496dcba..1723bfb 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -763,7 +763,38 @@ def OpGetDefaultQueue: Op<303, (outs ID:$res), (ins TYPE:$type),
def OpBuildNDRange: Op<304, (outs ID:$res), (ins TYPE:$type, ID:$GWS, ID:$LWS, ID:$GWO),
"$res = OpBuildNDRange $type $GWS $LWS $GWO">;
-// TODO: 3.42.23. Pipe Instructions
+// 3.42.23. Pipe Instructions
+
+def OpReadPipe: Op<274, (outs ID:$res), (ins TYPE:$type, ID:$Pipe, ID:$Pointer, ID:$PcktSize, ID:$PcktAlign),
+ "$res = OpReadPipe $type $Pipe $Pointer $PcktSize $PcktAlign">;
+def OpWritePipe: Op<275, (outs ID:$res), (ins TYPE:$type, ID:$Pipe, ID:$Pointer, ID:$PcktSize, ID:$PcktAlign),
+ "$res = OpWritePipe $type $Pipe $Pointer $PcktSize $PcktAlign">;
+def OpReservedReadPipe : Op<276, (outs ID:$res), (ins TYPE:$type, ID:$Pipe, ID:$ReserveId, ID:$Index, ID:$Pointer, ID:$PcktSize, ID:$PcktAlign),
+ "$res = OpReservedReadPipe $type $Pipe $ReserveId $Index $Pointer $PcktSize $PcktAlign">;
+def OpReservedWritePipe : Op<277, (outs ID:$res), (ins TYPE:$type, ID:$Pipe, ID:$ReserveId, ID:$Index, ID:$Pointer, ID:$PcktSize, ID:$PcktAlign),
+ "$res = OpReservedWritePipe $type $Pipe $ReserveId $Index $Pointer $PcktSize $PcktAlign">;
+def OpReserveReadPipePackets : Op<278, (outs ID:$res), (ins TYPE:$type, ID:$Pipe, ID:$NumPckts, ID:$PcktSize, ID:$PcktAlign),
+ "$res = OpReserveReadPipePackets $type $Pipe $NumPckts $PcktSize $PcktAlign">;
+def OpReserveWritePipePackets : Op<279, (outs ID:$res), (ins TYPE:$type, ID:$Pipe, ID:$NumPckts, ID:$PcktSize, ID:$PcktAlign),
+ "$res = OpReserveWritePipePackets $type $Pipe $NumPckts $PcktSize $PcktAlign">;
+def OpCommitReadPipe : Op<280, (outs), (ins ID:$Pipe, ID:$ReserveId, ID:$PcktSize, ID:$PcktAlign),
+ "OpCommitReadPipe $Pipe $ReserveId $PcktSize $PcktAlign">;
+def OpCommitWritePipe : Op<281, (outs), (ins ID:$Pipe, ID:$ReserveId, ID:$PcktSize, ID:$PcktAlign),
+ "OpCommitWritePipe $Pipe $ReserveId $PcktSize $PcktAlign">;
+def OpIsValidReserveId : Op<282, (outs ID:$res), (ins TYPE:$type, ID:$ReserveId),
+ "$res = OpIsValidReserveId $type $ReserveId">;
+def OpGetNumPipePackets : Op<283, (outs ID:$res), (ins TYPE:$type, ID:$Pipe, ID:$PacketSize, ID:$PacketAlign),
+ "$res = OpGetNumPipePackets $type $Pipe $PacketSize $PacketAlign">;
+def OpGetMaxPipePackets : Op<284, (outs ID:$res), (ins TYPE:$type, ID:$Pipe, ID:$PacketSize, ID:$PacketAlign),
+ "$res = OpGetMaxPipePackets $type $Pipe $PacketSize $PacketAlign">;
+def OpGroupReserveReadPipePackets : Op<285, (outs ID:$res), (ins TYPE:$type, ID:$Scope, ID:$Pipe, ID:$NumPckts, ID:$PacketSize, ID:$PacketAlign),
+ "$res = OpGroupReserveReadPipePackets $type $Scope $Pipe $NumPckts $PacketSize $PacketAlign">;
+def OpGroupReserveWritePipePackets : Op<286, (outs ID:$res), (ins TYPE:$type, ID:$Scope, ID:$Pipe, ID:$NumPckts, ID:$PacketSize, ID:$PacketAlign),
+ "$res = OpGroupReserveWritePipePackets $type $Scope $Pipe $NumPckts $PacketSize $PacketAlign">;
+def OpGroupCommitReadPipe : Op<287, (outs), (ins ID:$Scope, ID:$Pipe, ID:$ReserveId, ID:$PacketSize, ID:$PacketAlign),
+ "OpGroupCommitReadPipe $Scope $Pipe $ReserveId $PacketSize $PacketAlign">;
+def OpGroupCommitWritePipe : Op<288, (outs), (ins ID:$Scope, ID:$Pipe, ID:$ReserveId, ID:$PacketSize, ID:$PacketAlign),
+ "OpGroupCommitWritePipe $Scope $Pipe $ReserveId $PacketSize $PacketAlign">;
// 3.42.24. Non-Uniform Instructions
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index a1c6f79..81f1956 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -845,19 +845,10 @@ InstructionCost VPRegionBlock::cost(ElementCount VF, VPCostContext &Ctx) {
if (VF.isScalable())
return InstructionCost::getInvalid();
- // First compute the cost of the conditionally executed recipes, followed by
- // account for the branching cost, except if the mask is a header mask or
- // uniform condition.
- using namespace llvm::VPlanPatternMatch;
+ // Compute and return the cost of the conditionally executed recipes.
+ assert(VF.isVector() && "Can only compute vector cost at the moment.");
VPBasicBlock *Then = cast<VPBasicBlock>(getEntry()->getSuccessors()[0]);
- InstructionCost ThenCost = Then->cost(VF, Ctx);
-
- // For the scalar case, we may not always execute the original predicated
- // block, Thus, scale the block's cost by the probability of executing it.
- if (VF.isScalar())
- return ThenCost / getPredBlockCostDivisor(Ctx.CostKind);
-
- return ThenCost;
+ return Then->cost(VF, Ctx);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 58fab8f..5252e1f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2853,6 +2853,7 @@ void VPlanTransforms::replaceSymbolicStrides(
return R->getParent()->getParent() ||
R->getParent() == Plan.getVectorLoopRegion()->getSinglePredecessor();
};
+ ValueToSCEVMapTy RewriteMap;
for (const SCEV *Stride : StridesMap.values()) {
using namespace SCEVPatternMatch;
auto *StrideV = cast<SCEVUnknown>(Stride)->getValue();
@@ -2880,6 +2881,22 @@ void VPlanTransforms::replaceSymbolicStrides(
VPValue *CI = Plan.getOrAddLiveIn(ConstantInt::get(U->getType(), C));
StrideVPV->replaceUsesWithIf(CI, CanUseVersionedStride);
}
+ RewriteMap[StrideV] = PSE.getSCEV(StrideV);
+ }
+
+ for (VPRecipeBase &R : *Plan.getEntry()) {
+ auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
+ if (!ExpSCEV)
+ continue;
+ const SCEV *ScevExpr = ExpSCEV->getSCEV();
+ auto *NewSCEV =
+ SCEVParameterRewriter::rewrite(ScevExpr, *PSE.getSE(), RewriteMap);
+ if (NewSCEV != ScevExpr) {
+ VPValue *NewExp = vputils::getOrCreateVPValueForSCEVExpr(Plan, NewSCEV);
+ ExpSCEV->replaceAllUsesWith(NewExp);
+ if (Plan.getTripCount() == ExpSCEV)
+ Plan.resetTripCount(NewExp);
+ }
}
}
generated by cgit v1.1 at 2025-09-29 01:09:39 +0000