diff options
Diffstat (limited to 'llvm/lib/Target/SPIRV')
21 files changed, 308 insertions, 38 deletions
diff --git a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h index 78a066b..ed0a1e1 100644 --- a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h +++ b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h @@ -73,7 +73,11 @@ public: Entry(std::move(CR.Entry)), Exits(std::move(CR.Exits)), Blocks(std::move(CR.Blocks)) {} + ~ConvergenceRegion() { releaseMemory(); } + + ConvergenceRegion &operator=(ConvergenceRegion &&CR) = delete; ConvergenceRegion(const ConvergenceRegion &other) = delete; + ConvergenceRegion &operator=(const ConvergenceRegion &other) = delete; // Returns true if the given basic block belongs to this region, or to one of // its subregion. @@ -101,6 +105,9 @@ public: ~ConvergenceRegionInfo() { releaseMemory(); } + ConvergenceRegionInfo(const ConvergenceRegionInfo &LHS) = delete; + ConvergenceRegionInfo &operator=(const ConvergenceRegionInfo &LHS) = delete; + ConvergenceRegionInfo(ConvergenceRegionInfo &&LHS) : TopLevelRegion(LHS.TopLevelRegion) { if (TopLevelRegion != LHS.TopLevelRegion) { diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt index ba09451..6660de9 100644 --- a/llvm/lib/Target/SPIRV/CMakeLists.txt +++ b/llvm/lib/Target/SPIRV/CMakeLists.txt @@ -26,6 +26,7 @@ add_llvm_target(SPIRVCodeGen SPIRVGlobalRegistry.cpp SPIRVInstrInfo.cpp SPIRVInstructionSelector.cpp + SPIRVLegalizeImplicitBinding.cpp SPIRVStripConvergentIntrinsics.cpp SPIRVLegalizePointerCast.cpp SPIRVMergeRegionExitTargets.cpp diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVAsmBackend.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVAsmBackend.cpp index ef84d43..5710cf2 100644 --- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVAsmBackend.cpp +++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVAsmBackend.cpp @@ -21,8 +21,7 @@ public: SPIRVAsmBackend(llvm::endianness Endian) : MCAsmBackend(Endian) {} void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target, - MutableArrayRef<char> Data, uint64_t Value, - bool IsResolved) override {} + uint8_t *Data, uint64_t Value, bool IsResolved) override {} std::unique_ptr<MCObjectTargetWriter> createObjectTargetWriter() const override { diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp index a7f6fbc..4ec31bf 100644 --- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp +++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp @@ -96,7 +96,7 @@ void SPIRVInstPrinter::printOpConstantVarOps(const MCInst *MI, void SPIRVInstPrinter::recordOpExtInstImport(const MCInst *MI) { MCRegister Reg = MI->getOperand(0).getReg(); auto Name = getSPIRVStringOperand(*MI, 1); - auto Set = getExtInstSetFromString(Name); + auto Set = getExtInstSetFromString(std::move(Name)); ExtInstSetIDs.insert({Reg, Set}); } @@ -210,6 +210,7 @@ void SPIRVInstPrinter::printInst(const MCInst *MI, uint64_t Address, case SPIRV::OpConstantF: // The last fixed operand along with any variadic operands that follow // are part of the variable value. + assert(NumFixedOps > 0 && "Expected at least one fixed operand"); printOpConstantVarOps(MI, NumFixedOps - 1, OS); break; case SPIRV::OpCooperativeMatrixMulAddKHR: { @@ -375,7 +376,7 @@ void SPIRVInstPrinter::printOperand(const MCInst *MI, unsigned OpNo, if (Op.isReg()) O << '%' << (getIDFromRegister(Op.getReg().id()) + 1); else if (Op.isImm()) - O << formatImm((int64_t)Op.getImm()); + O << formatImm(Op.getImm()); else if (Op.isDFPImm()) O << formatImm((double)Op.getDFPImm()); else if (Op.isExpr()) diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h index 1688fa3..1934e98 100644 --- a/llvm/lib/Target/SPIRV/SPIRV.h +++ b/llvm/lib/Target/SPIRV/SPIRV.h @@ -23,6 +23,7 @@ ModulePass *createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM); FunctionPass *createSPIRVStructurizerPass(); FunctionPass *createSPIRVMergeRegionExitTargetsPass(); FunctionPass *createSPIRVStripConvergenceIntrinsicsPass(); +ModulePass *createSPIRVLegalizeImplicitBindingPass(); FunctionPass *createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM); FunctionPass *createSPIRVRegularizerPass(); FunctionPass *createSPIRVPreLegalizerCombiner(); @@ -49,6 +50,7 @@ void initializeSPIRVRegularizerPass(PassRegistry &); void initializeSPIRVMergeRegionExitTargetsPass(PassRegistry &); void initializeSPIRVPrepareFunctionsPass(PassRegistry &); void initializeSPIRVStripConvergentIntrinsicsPass(PassRegistry &); +void initializeSPIRVLegalizeImplicitBindingPass(PassRegistry &); } // namespace llvm #endif // LLVM_LIB_TARGET_SPIRV_SPIRV_H diff --git a/llvm/lib/Target/SPIRV/SPIRVAPI.cpp b/llvm/lib/Target/SPIRV/SPIRVAPI.cpp index cfe7ef4..d6581b2 100644 --- a/llvm/lib/Target/SPIRV/SPIRVAPI.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVAPI.cpp @@ -156,7 +156,7 @@ SPIRVTranslateModule(Module *M, std::string &SpirvObj, std::string &ErrMsg, } } return SPIRVTranslate(M, SpirvObj, ErrMsg, AllowExtNames, OLevel, - TargetTriple); + std::move(TargetTriple)); } } // namespace llvm diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp index 1ebfde2..c2a6e51 100644 --- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp @@ -50,7 +50,8 @@ class SPIRVAsmPrinter : public AsmPrinter { public: explicit SPIRVAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer) - : AsmPrinter(TM, std::move(Streamer), ID), ST(nullptr), TII(nullptr) {} + : AsmPrinter(TM, std::move(Streamer), ID), ModuleSectionsEmitted(false), + ST(nullptr), TII(nullptr), MAI(nullptr) {} static char ID; bool ModuleSectionsEmitted; const SPIRVSubtarget *ST; @@ -591,7 +592,9 @@ void SPIRVAsmPrinter::outputAnnotations(const Module &M) { cast<GlobalVariable>(CS->getOperand(1)->stripPointerCasts()); StringRef AnnotationString; - getConstantStringInfo(GV, AnnotationString); + [[maybe_unused]] bool Success = + getConstantStringInfo(GV, AnnotationString); + assert(Success && "Failed to get annotation string"); MCInst Inst; Inst.setOpcode(SPIRV::OpDecorate); Inst.addOperand(MCOperand::createReg(Reg)); diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp index 25cdf72..e6e86b7 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -51,7 +51,7 @@ struct IncomingCall { IncomingCall(const std::string BuiltinName, const DemangledBuiltin *Builtin, const Register ReturnRegister, const SPIRVType *ReturnType, const SmallVectorImpl<Register> &Arguments) - : BuiltinName(BuiltinName), Builtin(Builtin), + : BuiltinName(std::move(BuiltinName)), Builtin(Builtin), ReturnRegister(ReturnRegister), ReturnType(ReturnType), Arguments(Arguments) {} @@ -2619,6 +2619,7 @@ static bool generateConvertInst(const StringRef DemangledCall, GR->getSPIRVTypeID(Call->ReturnType)); } + assert(Builtin && "Conversion builtin not found."); if (Builtin->IsSaturated) buildOpDecorate(Call->ReturnRegister, MIRBuilder, SPIRV::Decoration::SaturatedConversion, {}); diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 947b574..f5a49e2 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -21,7 +21,9 @@ #include "llvm/IR/InstIterator.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/IntrinsicsSPIRV.h" +#include "llvm/IR/PatternMatch.h" #include "llvm/IR/TypedPointerType.h" +#include "llvm/Transforms/Utils/Local.h" #include <queue> #include <unordered_set> @@ -187,6 +189,8 @@ class SPIRVEmitIntrinsics void applyDemangledPtrArgTypes(IRBuilder<> &B); + GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP); + bool runOnFunction(Function &F); bool postprocessTypes(Module &M); bool processFunctionPointers(Module &M); @@ -495,7 +499,7 @@ void SPIRVEmitIntrinsics::propagateElemTypeRec( std::unordered_set<Value *> Visited; DenseMap<Function *, CallInst *> Ptrcasts; propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited, - Ptrcasts); + std::move(Ptrcasts)); } void SPIRVEmitIntrinsics::propagateElemTypeRec( @@ -893,17 +897,16 @@ Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper( bool Change = false; for (unsigned i = 0; i < U->getNumOperands(); ++i) { Value *Op = U->getOperand(i); + assert(Op && "Operands should not be null."); Type *OpTy = Op->getType(); Type *Ty = OpTy; - if (Op) { - if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) { - if (Type *NestedTy = - deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8)) - Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace()); - } else { - Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited, - UnknownElemTypeI8); - } + if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) { + if (Type *NestedTy = + deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8)) + Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace()); + } else { + Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited, + UnknownElemTypeI8); } Tys.push_back(Ty); Change |= Ty != OpTy; @@ -1458,6 +1461,24 @@ static void createSaturatedConversionDecoration(Instruction *I, createDecorationIntrinsic(I, SaturatedConversionNode, B); } +static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) { + if (auto *CI = dyn_cast<CallInst>(I)) { + if (Function *Fu = CI->getCalledFunction()) { + if (Fu->isIntrinsic()) { + unsigned const int IntrinsicId = Fu->getIntrinsicID(); + switch (IntrinsicId) { + case Intrinsic::fptosi_sat: + case Intrinsic::fptoui_sat: + createSaturatedConversionDecoration(I, B); + break; + default: + break; + } + } + } + } +} + Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) { if (!Call.isInlineAsm()) return &Call; @@ -2543,6 +2564,30 @@ void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) { } } +GetElementPtrInst * +SPIRVEmitIntrinsics::simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP) { + // getelementptr [0 x T], P, 0 (zero), I -> getelementptr T, P, I. + // If type is 0-length array and first index is 0 (zero), drop both the + // 0-length array type and the first index. This is a common pattern in the + // IR, e.g. when using a zero-length array as a placeholder for a flexible + // array such as unbound arrays. + assert(GEP && "GEP is null"); + Type *SrcTy = GEP->getSourceElementType(); + SmallVector<Value *, 8> Indices(GEP->indices()); + ArrayType *ArrTy = dyn_cast<ArrayType>(SrcTy); + if (ArrTy && ArrTy->getNumElements() == 0 && + PatternMatch::match(Indices[0], PatternMatch::m_Zero())) { + IRBuilder<> Builder(GEP); + Indices.erase(Indices.begin()); + SrcTy = ArrTy->getElementType(); + Value *NewGEP = Builder.CreateGEP(SrcTy, GEP->getPointerOperand(), Indices, + "", GEP->getNoWrapFlags()); + assert(llvm::isa<GetElementPtrInst>(NewGEP) && "NewGEP should be a GEP"); + return cast<GetElementPtrInst>(NewGEP); + } + return nullptr; +} + bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) { if (Func.isDeclaration()) return false; @@ -2560,14 +2605,30 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) { AggrConstTypes.clear(); AggrStores.clear(); - // fix GEP result types ahead of inference + // Fix GEP result types ahead of inference, and simplify if possible. + // Data structure for dead instructions that were simplified and replaced. + SmallPtrSet<Instruction *, 4> DeadInsts; for (auto &I : instructions(Func)) { auto *Ref = dyn_cast<GetElementPtrInst>(&I); if (!Ref || GR->findDeducedElementType(Ref)) continue; + + GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(Ref); + if (NewGEP) { + Ref->replaceAllUsesWith(NewGEP); + if (isInstructionTriviallyDead(Ref)) + DeadInsts.insert(Ref); + + Ref = NewGEP; + } if (Type *GepTy = getGEPType(Ref)) GR->addDeducedElementType(Ref, normalizeType(GepTy)); } + // Remove dead instructions that were simplified and replaced. + for (auto *I : DeadInsts) { + assert(I->use_empty() && "Dead instruction should not have any uses left"); + I->eraseFromParent(); + } processParamTypesByFunHeader(CurrF, B); @@ -2640,6 +2701,7 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) { if (isConvergenceIntrinsic(I)) continue; + addSaturatedDecorationToIntrinsic(I, B); processInstrAfterVisit(I, B); } diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp index 7f0d636..275463e 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp @@ -116,6 +116,7 @@ bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) { } } const NamedMDNode *ModuleFlags = M->getNamedMetadata("llvm.module.flags"); + assert(ModuleFlags && "Expected llvm.module.flags metadata to be present"); for (const auto *Op : ModuleFlags->operands()) { const MDOperand &MaybeStrOp = Op->getOperand(1); if (MaybeStrOp.equalsStr("Dwarf Version")) diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 83fccdc..cfe24c8 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -87,7 +87,7 @@ storageClassRequiresExplictLayout(SPIRV::StorageClass::StorageClass SC) { } SPIRVGlobalRegistry::SPIRVGlobalRegistry(unsigned PointerSize) - : PointerSize(PointerSize), Bound(0) {} + : PointerSize(PointerSize), Bound(0), CurMF(nullptr) {} SPIRVType *SPIRVGlobalRegistry::assignIntTypeToVReg(unsigned BitWidth, Register VReg, @@ -474,8 +474,8 @@ Register SPIRVGlobalRegistry::getOrCreateBaseRegister( } if (Type->getOpcode() == SPIRV::OpTypeFloat) { SPIRVType *SpvBaseType = getOrCreateSPIRVFloatType(BitWidth, I, TII); - return getOrCreateConstFP(dyn_cast<ConstantFP>(Val)->getValue(), I, - SpvBaseType, TII, ZeroAsNull); + return getOrCreateConstFP(cast<ConstantFP>(Val)->getValue(), I, SpvBaseType, + TII, ZeroAsNull); } assert(Type->getOpcode() == SPIRV::OpTypeInt); SPIRVType *SpvBaseType = getOrCreateSPIRVIntegerType(BitWidth, I, TII); @@ -828,6 +828,8 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeArray(uint32_t NumElems, "Invalid array element type"); SPIRVType *SpvTypeInt32 = getOrCreateSPIRVIntegerType(32, MIRBuilder); SPIRVType *ArrayType = nullptr; + const SPIRVSubtarget &ST = + cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget()); if (NumElems != 0) { Register NumElementsVReg = buildConstantInt(NumElems, MIRBuilder, SpvTypeInt32, EmitIR); @@ -838,6 +840,10 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeArray(uint32_t NumElems, .addUse(NumElementsVReg); }); } else { + assert(ST.isShader() && "Runtime arrays are not allowed in non-shader " + "SPIR-V modules."); + if (!ST.isShader()) + return nullptr; ArrayType = createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) { return MIRBuilder.buildInstr(SPIRV::OpTypeRuntimeArray) .addDef(createTypeVReg(MIRBuilder)) @@ -1063,7 +1069,8 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType( MIRBuilder); }; } - return getOpTypeStruct(SType, MIRBuilder, AccQual, Decorator, EmitIR); + return getOpTypeStruct(SType, MIRBuilder, AccQual, std::move(Decorator), + EmitIR); } if (auto FType = dyn_cast<FunctionType>(Ty)) { SPIRVType *RetTy = findSPIRVType(FType->getReturnType(), MIRBuilder, @@ -1400,8 +1407,9 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateLayoutType( // We need a new OpTypeStruct instruction because decorations will be // different from a struct with an explicit layout created from a different // entry point. - SPIRVType *SPIRVStructType = getOpTypeStruct( - ST, MIRBuilder, SPIRV::AccessQualifier::None, Decorator, EmitIr); + SPIRVType *SPIRVStructType = + getOpTypeStruct(ST, MIRBuilder, SPIRV::AccessQualifier::None, + std::move(Decorator), EmitIr); add(Key, SPIRVStructType); return SPIRVStructType; } diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index d4fa62a..5259db1 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -362,6 +362,7 @@ SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, const RegisterBankInfo &RBI) : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), + MRI(nullptr), #define GET_GLOBALISEL_PREDICATES_INIT #include "SPIRVGenGlobalISel.inc" #undef GET_GLOBALISEL_PREDICATES_INIT @@ -665,6 +666,11 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, case TargetOpcode::G_FPTOUI: return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); + case TargetOpcode::G_FPTOSI_SAT: + return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); + case TargetOpcode::G_FPTOUI_SAT: + return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); + case TargetOpcode::G_SITOFP: return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); case TargetOpcode::G_UITOFP: @@ -3569,7 +3575,7 @@ bool SPIRVInstructionSelector::selectFirstBitSet64Overflow( // Join all the resulting registers back into the return type in order // (ie i32x2, i32x2, i32x1 -> i32x5) - return selectOpWithSrcs(ResVReg, ResType, I, PartialRegs, + return selectOpWithSrcs(ResVReg, ResType, I, std::move(PartialRegs), SPIRV::OpCompositeConstruct); } diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizeImplicitBinding.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizeImplicitBinding.cpp new file mode 100644 index 0000000..0398e52 --- /dev/null +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizeImplicitBinding.cpp @@ -0,0 +1,159 @@ +//===- SPIRVLegalizeImplicitBinding.cpp - Legalize implicit bindings ----*- C++ +//-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass legalizes the @llvm.spv.resource.handlefromimplicitbinding +// intrinsic by replacing it with a call to +// @llvm.spv.resource.handlefrombinding. +// +//===----------------------------------------------------------------------===// + +#include "SPIRV.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InstVisitor.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsSPIRV.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" +#include <algorithm> +#include <vector> + +using namespace llvm; + +namespace { +class SPIRVLegalizeImplicitBinding : public ModulePass { +public: + static char ID; + SPIRVLegalizeImplicitBinding() : ModulePass(ID) {} + + bool runOnModule(Module &M) override; + +private: + void collectBindingInfo(Module &M); + uint32_t getAndReserveFirstUnusedBinding(uint32_t DescSet); + void replaceImplicitBindingCalls(Module &M); + + // A map from descriptor set to a bit vector of used binding numbers. + std::vector<BitVector> UsedBindings; + // A list of all implicit binding calls, to be sorted by order ID. + SmallVector<CallInst *, 16> ImplicitBindingCalls; +}; + +struct BindingInfoCollector : public InstVisitor<BindingInfoCollector> { + std::vector<BitVector> &UsedBindings; + SmallVector<CallInst *, 16> &ImplicitBindingCalls; + + BindingInfoCollector(std::vector<BitVector> &UsedBindings, + SmallVector<CallInst *, 16> &ImplicitBindingCalls) + : UsedBindings(UsedBindings), ImplicitBindingCalls(ImplicitBindingCalls) { + } + + void visitCallInst(CallInst &CI) { + if (CI.getIntrinsicID() == Intrinsic::spv_resource_handlefrombinding) { + const uint32_t DescSet = + cast<ConstantInt>(CI.getArgOperand(0))->getZExtValue(); + const uint32_t Binding = + cast<ConstantInt>(CI.getArgOperand(1))->getZExtValue(); + + if (UsedBindings.size() <= DescSet) { + UsedBindings.resize(DescSet + 1); + UsedBindings[DescSet].resize(64); + } + if (UsedBindings[DescSet].size() <= Binding) { + UsedBindings[DescSet].resize(2 * Binding + 1); + } + UsedBindings[DescSet].set(Binding); + } else if (CI.getIntrinsicID() == + Intrinsic::spv_resource_handlefromimplicitbinding) { + ImplicitBindingCalls.push_back(&CI); + } + } +}; + +void SPIRVLegalizeImplicitBinding::collectBindingInfo(Module &M) { + BindingInfoCollector InfoCollector(UsedBindings, ImplicitBindingCalls); + InfoCollector.visit(M); + + // Sort the collected calls by their order ID. + std::sort( + ImplicitBindingCalls.begin(), ImplicitBindingCalls.end(), + [](const CallInst *A, const CallInst *B) { + const uint32_t OrderIdArgIdx = 0; + const uint32_t OrderA = + cast<ConstantInt>(A->getArgOperand(OrderIdArgIdx))->getZExtValue(); + const uint32_t OrderB = + cast<ConstantInt>(B->getArgOperand(OrderIdArgIdx))->getZExtValue(); + return OrderA < OrderB; + }); +} + +uint32_t SPIRVLegalizeImplicitBinding::getAndReserveFirstUnusedBinding( + uint32_t DescSet) { + if (UsedBindings.size() <= DescSet) { + UsedBindings.resize(DescSet + 1); + UsedBindings[DescSet].resize(64); + } + + int NewBinding = UsedBindings[DescSet].find_first_unset(); + if (NewBinding == -1) { + NewBinding = UsedBindings[DescSet].size(); + UsedBindings[DescSet].resize(2 * NewBinding + 1); + } + + UsedBindings[DescSet].set(NewBinding); + return NewBinding; +} + +void SPIRVLegalizeImplicitBinding::replaceImplicitBindingCalls(Module &M) { + for (CallInst *OldCI : ImplicitBindingCalls) { + IRBuilder<> Builder(OldCI); + const uint32_t DescSet = + cast<ConstantInt>(OldCI->getArgOperand(1))->getZExtValue(); + const uint32_t NewBinding = getAndReserveFirstUnusedBinding(DescSet); + + SmallVector<Value *, 8> Args; + Args.push_back(Builder.getInt32(DescSet)); + Args.push_back(Builder.getInt32(NewBinding)); + + // Copy the remaining arguments from the old call. + for (uint32_t i = 2; i < OldCI->arg_size(); ++i) { + Args.push_back(OldCI->getArgOperand(i)); + } + + Function *NewFunc = Intrinsic::getOrInsertDeclaration( + &M, Intrinsic::spv_resource_handlefrombinding, OldCI->getType()); + CallInst *NewCI = Builder.CreateCall(NewFunc, Args); + NewCI->setCallingConv(OldCI->getCallingConv()); + + OldCI->replaceAllUsesWith(NewCI); + OldCI->eraseFromParent(); + } +} + +bool SPIRVLegalizeImplicitBinding::runOnModule(Module &M) { + collectBindingInfo(M); + if (ImplicitBindingCalls.empty()) { + return false; + } + + replaceImplicitBindingCalls(M); + return true; +} +} // namespace + +char SPIRVLegalizeImplicitBinding::ID = 0; + +INITIALIZE_PASS(SPIRVLegalizeImplicitBinding, "legalize-spirv-implicit-binding", + "Legalize SPIR-V implicit bindings", false, false) + +ModulePass *llvm::createSPIRVLegalizeImplicitBindingPass() { + return new SPIRVLegalizeImplicitBinding(); +}
\ No newline at end of file diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp index 1995e0f..170bddd 100644 --- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp @@ -203,6 +203,10 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) { .legalForCartesianProduct(allIntScalarsAndVectors, allFloatScalarsAndVectors); + getActionDefinitionsBuilder({G_FPTOSI_SAT, G_FPTOUI_SAT}) + .legalForCartesianProduct(allIntScalarsAndVectors, + allFloatScalarsAndVectors); + getActionDefinitionsBuilder({G_SITOFP, G_UITOFP}) .legalForCartesianProduct(allFloatScalarsAndVectors, allScalarsAndVectors); diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp index 0cd9d78..8039cf0 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -93,7 +93,7 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category, if (Reqs.isCapabilityAvailable(Cap)) { ReqExts.append(getSymbolicOperandExtensions( SPIRV::OperandCategory::CapabilityOperand, Cap)); - return {true, {Cap}, ReqExts, ReqMinVer, ReqMaxVer}; + return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer}; } } else { // By SPIR-V specification: "If an instruction, enumerant, or other @@ -111,7 +111,7 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category, if (i == Sz - 1 || !AvoidCaps.S.contains(Cap)) { ReqExts.append(getSymbolicOperandExtensions( SPIRV::OperandCategory::CapabilityOperand, Cap)); - return {true, {Cap}, ReqExts, ReqMinVer, ReqMaxVer}; + return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer}; } } } @@ -558,7 +558,7 @@ static void collectOtherInstr(MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI, bool Append = true) { MAI.setSkipEmission(&MI); InstrSignature MISign = instrToSignature(MI, MAI, true); - auto FoundMI = IS.insert(MISign); + auto FoundMI = IS.insert(std::move(MISign)); if (!FoundMI.second) return; // insert failed, so we found a duplicate; don't add it to MAI.MS // No duplicates, so add it. @@ -744,8 +744,14 @@ void SPIRV::RequirementHandler::checkSatisfiable( IsSatisfiable = false; } + AvoidCapabilitiesSet AvoidCaps; + if (!ST.isShader()) + AvoidCaps.S.insert(SPIRV::Capability::Shader); + else + AvoidCaps.S.insert(SPIRV::Capability::Kernel); + for (auto Cap : MinimalCaps) { - if (AvailableCaps.contains(Cap)) + if (AvailableCaps.contains(Cap) && !AvoidCaps.S.contains(Cap)) continue; LLVM_DEBUG(dbgs() << "Capability not supported: " << getSymbolicOperandMnemonic( @@ -1865,6 +1871,11 @@ void addInstrRequirements(const MachineInstr &MI, Reqs.addCapability(SPIRV::Capability::TernaryBitwiseFunctionINTEL); break; } + case SPIRV::OpCopyMemorySized: { + Reqs.addCapability(SPIRV::Capability::Addresses); + // TODO: Add UntypedPointersKHR when implemented. + break; + } default: break; diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h index a0d47cb..41c792a 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h @@ -54,8 +54,8 @@ struct Requirements { std::optional<Capability::Capability> Cap = {}, ExtensionList Exts = {}, VersionTuple MinVer = VersionTuple(), VersionTuple MaxVer = VersionTuple()) - : IsSatisfiable(IsSatisfiable), Cap(Cap), Exts(Exts), MinVer(MinVer), - MaxVer(MaxVer) {} + : IsSatisfiable(IsSatisfiable), Cap(Cap), Exts(std::move(Exts)), + MinVer(MinVer), MaxVer(MaxVer) {} Requirements(Capability::Capability Cap) : Requirements(true, {Cap}) {} }; @@ -217,7 +217,8 @@ struct SPIRVModuleAnalysis : public ModulePass { static char ID; public: - SPIRVModuleAnalysis() : ModulePass(ID) {} + SPIRVModuleAnalysis() + : ModulePass(ID), ST(nullptr), GR(nullptr), TII(nullptr), MMI(nullptr) {} bool runOnModule(Module &M) override; void getAnalysisUsage(AnalysisUsage &AU) const override; diff --git a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp index 1d38244..d17528d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp @@ -147,7 +147,7 @@ void visit(MachineFunction &MF, MachineBasicBlock &Start, // Do a preorder traversal of the CFG starting from the given function's entry // point. Calls |op| on each basic block encountered during the traversal. void visit(MachineFunction &MF, std::function<void(MachineBasicBlock *)> op) { - visit(MF, *MF.begin(), op); + visit(MF, *MF.begin(), std::move(op)); } bool SPIRVPostLegalizer::runOnMachineFunction(MachineFunction &MF) { diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index f4b4846..b62db7f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -99,6 +99,7 @@ addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR, SPIRVType *ExtType = GR->getOrCreateSPIRVType( Const->getType(), MIB, SPIRV::AccessQualifier::ReadWrite, true); + assert(SrcMI && "Expected source instruction to be valid"); SrcMI->setDesc(STI.getInstrInfo()->get(SPIRV::OpConstantNull)); SrcMI->addOperand(MachineOperand::CreateReg( GR->getSPIRVTypeID(ExtType), false)); diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp index 595424b..74aec4f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp @@ -234,7 +234,7 @@ static SmallVector<Metadata *> parseAnnotation(Value *I, return SmallVector<Metadata *>{}; MDs.push_back(MDNode::get(Ctx, MDsItem)); } - return Pos == static_cast<int>(Anno.length()) ? MDs + return Pos == static_cast<int>(Anno.length()) ? std::move(MDs) : SmallVector<Metadata *>{}; } diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp index d7cf211..e0bfb77 100644 --- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp @@ -226,6 +226,7 @@ void SPIRVPassConfig::addIRPasses() { } void SPIRVPassConfig::addISelPrepare() { + addPass(createSPIRVLegalizeImplicitBindingPass()); addPass(createSPIRVEmitIntrinsicsPass(&getTM<SPIRVTargetMachine>())); if (TM.getSubtargetImpl()->isLogicalSPIRV()) addPass(createSPIRVLegalizePointerCastPass(&getTM<SPIRVTargetMachine>())); diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp index 416d811..820e56b 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp @@ -463,8 +463,10 @@ std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) { DemangledNameLenStart = NameSpaceStart + 11; } Start = Name.find_first_not_of("0123456789", DemangledNameLenStart); - Name.substr(DemangledNameLenStart, Start - DemangledNameLenStart) - .getAsInteger(10, Len); + [[maybe_unused]] bool Error = + Name.substr(DemangledNameLenStart, Start - DemangledNameLenStart) + .getAsInteger(10, Len); + assert(!Error && "Failed to parse demangled name length"); return Name.substr(Start, Len).str(); } @@ -756,7 +758,7 @@ bool getVacantFunctionName(Module &M, std::string &Name) { for (unsigned I = 0; I < MaxIters; ++I) { std::string OrdName = Name + Twine(I).str(); if (!M.getFunction(OrdName)) { - Name = OrdName; + Name = std::move(OrdName); return true; } } |