diff options
Diffstat (limited to 'llvm/lib/Target')
19 files changed, 132 insertions, 26 deletions
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 76a790dc..8457f61 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -22308,6 +22308,37 @@ static SDValue performExtBinopLoadFold(SDNode *N, SelectionDAG &DAG) { return DAG.getNode(N->getOpcode(), DL, VT, Ext0, NShift); } +// Attempt to combine the following patterns: +// SUB x, (CSET LO, (CMP a, b)) -> SBC x, 0, (CMP a, b) +// SUB (SUB x, y), (CSET LO, (CMP a, b)) -> SBC x, y, (CMP a, b) +// The CSET may be preceded by a ZEXT. +static SDValue performSubWithBorrowCombine(SDNode *N, SelectionDAG &DAG) { + if (N->getOpcode() != ISD::SUB) + return SDValue(); + + EVT VT = N->getValueType(0); + if (VT != MVT::i32 && VT != MVT::i64) + return SDValue(); + + SDValue N1 = N->getOperand(1); + if (N1.getOpcode() == ISD::ZERO_EXTEND && N1.hasOneUse()) + N1 = N1.getOperand(0); + if (!N1.hasOneUse() || getCSETCondCode(N1) != AArch64CC::LO) + return SDValue(); + + SDValue Flags = N1.getOperand(3); + if (Flags.getOpcode() != AArch64ISD::SUBS) + return SDValue(); + + SDLoc DL(N); + SDValue N0 = N->getOperand(0); + if (N0->getOpcode() == ISD::SUB) + return DAG.getNode(AArch64ISD::SBC, DL, VT, N0.getOperand(0), + N0.getOperand(1), Flags); + return DAG.getNode(AArch64ISD::SBC, DL, VT, N0, DAG.getConstant(0, DL, VT), + Flags); +} + static SDValue performAddSubCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { // Try to change sum of two reductions. @@ -22329,6 +22360,8 @@ static SDValue performAddSubCombine(SDNode *N, return Val; if (SDValue Val = performAddSubIntoVectorOp(N, DCI.DAG)) return Val; + if (SDValue Val = performSubWithBorrowCombine(N, DCI.DAG)) + return Val; if (SDValue Val = performExtBinopLoadFold(N, DCI.DAG)) return Val; diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 76f076a..b30e3d0 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -4444,6 +4444,11 @@ defm PRFUM : PrefetchUnscaled<0b11, 0, 0b10, "prfum", [(AArch64Prefetch timm:$Rt, (am_unscaled64 GPR64sp:$Rn, simm9:$offset))]>; +// PRFM falls back to PRFUM for negative or unaligned offsets (not a multiple +// of 8). +def : InstAlias<"prfm $Rt, [$Rn, $offset]", + (PRFUMi prfop:$Rt, GPR64sp:$Rn, simm9_offset_fb64:$offset), 0>; + //--- // (unscaled immediate, unprivileged) defm LDTRX : LoadUnprivileged<0b11, 0, 0b01, GPR64, "ldtr">; diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index e1f4386..65b6077 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -3597,6 +3597,18 @@ let Predicates = [HasSVE_or_SME] in { def : Pat<(sext (i32 (vector_extract nxv4i32:$vec, VectorIndexS:$index))), (SMOVvi32to64 (v4i32 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexS:$index)>; + + // Extracts of ``unsigned'' i8 or i16 elements lead to the zero-extend being + // transformed to an AND mask. The mask is redundant since UMOV already zeroes + // the high bits of the destination register. + def : Pat<(i32 (and (vector_extract nxv16i8:$vec, VectorIndexB:$index), 0xff)), + (UMOVvi8 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index)>; + def : Pat<(i32 (and (vector_extract nxv8i16:$vec, VectorIndexH:$index), 0xffff)), + (UMOVvi16 (v8i16 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexH:$index)>; + def : Pat<(i64 (and (i64 (anyext (i32 (vector_extract nxv16i8:$vec, VectorIndexB:$index)))), (i64 0xff))), + (SUBREG_TO_REG (i64 0), (i32 (UMOVvi8 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index)), sub_32)>; + def : Pat<(i64 (and (i64 (anyext (i32 (vector_extract nxv8i16:$vec, VectorIndexH:$index)))), (i64 0xffff))), + (SUBREG_TO_REG (i64 0), (i32 (UMOVvi16 (v8i16 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexH:$index)), sub_32)>; } // End HasNEON // Extract first element from vector. diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp index 068954f..0bf2b31 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -54,7 +54,6 @@ #include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h" #include <memory> #include <optional> -#include <string> using namespace llvm; diff --git a/llvm/lib/Target/AVR/AVRTargetTransformInfo.h b/llvm/lib/Target/AVR/AVRTargetTransformInfo.h index 0daeeb8..338a7c8 100644 --- a/llvm/lib/Target/AVR/AVRTargetTransformInfo.h +++ b/llvm/lib/Target/AVR/AVRTargetTransformInfo.h @@ -21,7 +21,6 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/BasicTTIImpl.h" #include "llvm/IR/Function.h" -#include <optional> namespace llvm { diff --git a/llvm/lib/Target/CSKY/CSKYConstantIslandPass.cpp b/llvm/lib/Target/CSKY/CSKYConstantIslandPass.cpp index 7885d93..a2cf0a5 100644 --- a/llvm/lib/Target/CSKY/CSKYConstantIslandPass.cpp +++ b/llvm/lib/Target/CSKY/CSKYConstantIslandPass.cpp @@ -48,7 +48,6 @@ #include "llvm/Support/Format.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <cassert> #include <cstdint> #include <iterator> diff --git a/llvm/lib/Target/DirectX/DXContainerGlobals.cpp b/llvm/lib/Target/DirectX/DXContainerGlobals.cpp index 677203d..95577dd 100644 --- a/llvm/lib/Target/DirectX/DXContainerGlobals.cpp +++ b/llvm/lib/Target/DirectX/DXContainerGlobals.cpp @@ -29,7 +29,6 @@ #include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Utils/ModuleUtils.h" #include <cstdint> -#include <optional> using namespace llvm; using namespace llvm::dxil; diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.h b/llvm/lib/Target/DirectX/DXILRootSignature.h index b990b6c..ec82aa9 100644 --- a/llvm/lib/Target/DirectX/DXILRootSignature.h +++ b/llvm/lib/Target/DirectX/DXILRootSignature.h @@ -21,7 +21,6 @@ #include "llvm/IR/PassManager.h" #include "llvm/MC/DXContainerRootSignature.h" #include "llvm/Pass.h" -#include <optional> namespace llvm { namespace dxil { diff --git a/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.h b/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.h index f2c00c7..7cbc092 100644 --- a/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.h +++ b/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.h @@ -19,7 +19,6 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/MemoryBufferRef.h" #include <memory> -#include <string> #include <vector> namespace llvm { diff --git a/llvm/lib/Target/M68k/M68kSubtarget.h b/llvm/lib/Target/M68k/M68kSubtarget.h index 16ca7d2..4f96858 100644 --- a/llvm/lib/Target/M68k/M68kSubtarget.h +++ b/llvm/lib/Target/M68k/M68kSubtarget.h @@ -27,8 +27,6 @@ #include "llvm/MC/MCInstrItineraries.h" #include "llvm/Support/Alignment.h" -#include <string> - #define GET_SUBTARGETINFO_HEADER #include "M68kGenSubtargetInfo.inc" diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.h b/llvm/lib/Target/PowerPC/PPCSubtarget.h index f275802..7d93358 100644 --- a/llvm/lib/Target/PowerPC/PPCSubtarget.h +++ b/llvm/lib/Target/PowerPC/PPCSubtarget.h @@ -23,7 +23,6 @@ #include "llvm/IR/DataLayout.h" #include "llvm/MC/MCInstrItineraries.h" #include "llvm/TargetParser/Triple.h" -#include <string> #define GET_SUBTARGETINFO_HEADER #include "PPCGenSubtargetInfo.inc" diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index 5b72334..0b964c4 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -956,6 +956,9 @@ def FeatureStdExtSsdbltrp def FeatureStdExtSmepmp : RISCVExtension<1, 0, "Enhanced Physical Memory Protection">; +def FeatureStdExtSmpmpmt + : RISCVExperimentalExtension<0, 6, "PMP-based Memory Types Extension">; + def FeatureStdExtSmrnmi : RISCVExtension<1, 0, "Resumable Non-Maskable Interrupts">; def HasStdExtSmrnmi : Predicate<"Subtarget->hasStdExtSmrnmi()">, diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 332433b..3d8eb40 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1683,7 +1683,8 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, !TypeSize::isKnownLE(DL.getTypeSizeInBits(Src), SrcLT.second.getSizeInBits()) || !TypeSize::isKnownLE(DL.getTypeSizeInBits(Dst), - DstLT.second.getSizeInBits())) + DstLT.second.getSizeInBits()) || + SrcLT.first > 1 || DstLT.first > 1) return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); // The split cost is handled by the base getCastInstrCost diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp index 0175f2f..970b83d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp @@ -612,13 +612,10 @@ void SPIRVAsmPrinter::outputExecutionMode(const Module &M) { // Collect the SPIRVTypes for fp16, fp32, and fp64 and the constant of // type int32 with 0 value to represent the FP Fast Math Mode. std::vector<const MachineInstr *> SPIRVFloatTypes; - const MachineInstr *ConstZero = nullptr; + const MachineInstr *ConstZeroInt32 = nullptr; for (const MachineInstr *MI : MAI->getMSInstrs(SPIRV::MB_TypeConstVars)) { - // Skip if the instruction is not OpTypeFloat or OpConstant. unsigned OpCode = MI->getOpcode(); - if (OpCode != SPIRV::OpTypeFloat && OpCode != SPIRV::OpConstantNull) - continue; // Collect the SPIRV type if it's a float. if (OpCode == SPIRV::OpTypeFloat) { @@ -629,14 +626,18 @@ void SPIRVAsmPrinter::outputExecutionMode(const Module &M) { continue; } SPIRVFloatTypes.push_back(MI); - } else { + continue; + } + + if (OpCode == SPIRV::OpConstantNull) { // Check if the constant is int32, if not skip it. const MachineRegisterInfo &MRI = MI->getMF()->getRegInfo(); MachineInstr *TypeMI = MRI.getVRegDef(MI->getOperand(1).getReg()); - if (!TypeMI || TypeMI->getOperand(1).getImm() != 32) - continue; - - ConstZero = MI; + bool IsInt32Ty = TypeMI && + TypeMI->getOpcode() == SPIRV::OpTypeInt && + TypeMI->getOperand(1).getImm() == 32; + if (IsInt32Ty) + ConstZeroInt32 = MI; } } @@ -657,9 +658,9 @@ void SPIRVAsmPrinter::outputExecutionMode(const Module &M) { MCRegister TypeReg = MAI->getRegisterAlias(MF, MI->getOperand(0).getReg()); Inst.addOperand(MCOperand::createReg(TypeReg)); - assert(ConstZero && "There should be a constant zero."); + assert(ConstZeroInt32 && "There should be a constant zero."); MCRegister ConstReg = MAI->getRegisterAlias( - ConstZero->getMF(), ConstZero->getOperand(0).getReg()); + ConstZeroInt32->getMF(), ConstZeroInt32->getOperand(0).getReg()); Inst.addOperand(MCOperand::createReg(ConstReg)); outputMCInst(Inst); } diff --git a/llvm/lib/Target/Sparc/SparcSubtarget.h b/llvm/lib/Target/Sparc/SparcSubtarget.h index b1decca..f575f6d 100644 --- a/llvm/lib/Target/Sparc/SparcSubtarget.h +++ b/llvm/lib/Target/Sparc/SparcSubtarget.h @@ -21,7 +21,6 @@ #include "llvm/IR/DataLayout.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/TargetParser/Triple.h" -#include <string> #define GET_SUBTARGETINFO_HEADER #include "SparcGenSubtargetInfo.inc" diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp index 92a9812..70f7b88 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp @@ -119,18 +119,82 @@ InstructionCost WebAssemblyTTIImpl::getCastInstrCost( } } - // extend_low static constexpr TypeConversionCostTblEntry ConversionTbl[] = { + // extend_low {ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i32, 1}, {ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i32, 1}, {ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i16, 1}, {ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i16, 1}, {ISD::SIGN_EXTEND, MVT::v8i16, MVT::v8i8, 1}, {ISD::ZERO_EXTEND, MVT::v8i16, MVT::v8i8, 1}, + // 2 x extend_low {ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i16, 2}, {ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i16, 2}, {ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i8, 2}, {ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i8, 2}, + // extend_low, extend_high + {ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, 2}, + {ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, 2}, + {ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 2}, + {ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 2}, + {ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 2}, + {ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 2}, + // 2x extend_low, extend_high + {ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i32, 4}, + {ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i32, 4}, + {ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i16, 4}, + {ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i16, 4}, + // shuffle + {ISD::TRUNCATE, MVT::v2i16, MVT::v2i32, 2}, + {ISD::TRUNCATE, MVT::v2i8, MVT::v2i32, 4}, + {ISD::TRUNCATE, MVT::v4i16, MVT::v4i32, 2}, + {ISD::TRUNCATE, MVT::v4i8, MVT::v4i32, 4}, + // narrow, and + {ISD::TRUNCATE, MVT::v8i16, MVT::v8i32, 2}, + {ISD::TRUNCATE, MVT::v8i8, MVT::v8i16, 2}, + // narrow, 2x and + {ISD::TRUNCATE, MVT::v16i8, MVT::v16i16, 3}, + // 3x narrow, 4x and + {ISD::TRUNCATE, MVT::v8i16, MVT::v8i64, 7}, + {ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, 7}, + // 7x narrow, 8x and + {ISD::TRUNCATE, MVT::v16i8, MVT::v16i64, 15}, + // convert_i32x4 + {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i32, 1}, + {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, 1}, + {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 1}, + {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 1}, + // extend_low, convert + {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i16, 2}, + {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i16, 2}, + {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i16, 2}, + {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i16, 2}, + // extend_low x 2, convert + {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i8, 3}, + {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i8, 3}, + {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i8, 3}, + {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i8, 3}, + // several shuffles + {ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i8, 10}, + {ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i8, 10}, + {ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, 10}, + {ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i8, 10}, + /// trunc_sat, const, and, 3x narrow + {ISD::FP_TO_SINT, MVT::v2i8, MVT::v2f32, 6}, + {ISD::FP_TO_UINT, MVT::v2i8, MVT::v2f32, 6}, + {ISD::FP_TO_SINT, MVT::v4i8, MVT::v4f32, 6}, + {ISD::FP_TO_UINT, MVT::v4i8, MVT::v4f32, 6}, + /// trunc_sat, const, and, narrow + {ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f32, 4}, + {ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f32, 4}, + {ISD::FP_TO_SINT, MVT::v4i16, MVT::v4f32, 4}, + {ISD::FP_TO_UINT, MVT::v4i16, MVT::v4f32, 4}, + // 2x trunc_sat, const, 2x and, 3x narrow + {ISD::FP_TO_SINT, MVT::v8i8, MVT::v8f32, 8}, + {ISD::FP_TO_UINT, MVT::v8i8, MVT::v8f32, 8}, + // 2x trunc_sat, const, 2x and, narrow + {ISD::FP_TO_SINT, MVT::v8i16, MVT::v8f32, 6}, + {ISD::FP_TO_UINT, MVT::v8i16, MVT::v8f32, 6}, }; if (const auto *Entry = diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h index 2573066..4146c0e 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h @@ -21,7 +21,6 @@ #include "WebAssemblyTargetMachine.h" #include "llvm/CodeGen/BasicTTIImpl.h" -#include <algorithm> namespace llvm { diff --git a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp index ab6e6d0..b3bf37a 100644 --- a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp +++ b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp @@ -50,7 +50,6 @@ #include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <cassert> #include <iterator> #include <utility> diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 0c2bd7c..d4ad98a 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -50,7 +50,6 @@ #include "llvm/Transforms/CFGuard.h" #include <memory> #include <optional> -#include <string> using namespace llvm; |
