aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--llvm/include/llvm/CodeGen/BasicTTIImpl.h19
-rw-r--r--llvm/include/llvm/IR/Instructions.h70
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp2
-rw-r--r--llvm/lib/IR/Instructions.cpp75
-rw-r--r--llvm/lib/Target/AArch64/AArch64ISelLowering.cpp5
-rw-r--r--llvm/lib/Target/ARM/ARMISelLowering.cpp2
-rw-r--r--llvm/lib/Target/RISCV/RISCVISelLowering.cpp3
-rw-r--r--llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp7
-rw-r--r--llvm/lib/Target/X86/X86TargetTransformInfo.cpp2
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp4
-rw-r--r--llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp74
-rw-r--r--llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll31
-rw-r--r--llvm/unittests/IR/InstructionsTest.cpp178
-rw-r--r--llvm/unittests/IR/ShuffleVectorInstTest.cpp71
14 files changed, 325 insertions, 218 deletions
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index c11d558..8508086 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -935,31 +935,28 @@ public:
ArrayRef<int> Mask,
VectorType *Ty, int &Index,
VectorType *&SubTy) const {
- int Limit = Mask.size() * 2;
- if (Mask.empty() ||
- // Extra check required by isSingleSourceMaskImpl function (called by
- // ShuffleVectorInst::isSingleSourceMask).
- any_of(Mask, [Limit](int I) { return I >= Limit; }))
+ if (Mask.empty())
return Kind;
+ int NumSrcElts = Ty->getElementCount().getKnownMinValue();
switch (Kind) {
case TTI::SK_PermuteSingleSrc:
- if (ShuffleVectorInst::isReverseMask(Mask))
+ if (ShuffleVectorInst::isReverseMask(Mask, NumSrcElts))
return TTI::SK_Reverse;
- if (ShuffleVectorInst::isZeroEltSplatMask(Mask))
+ if (ShuffleVectorInst::isZeroEltSplatMask(Mask, NumSrcElts))
return TTI::SK_Broadcast;
break;
case TTI::SK_PermuteTwoSrc: {
int NumSubElts;
if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
- Mask, Mask.size(), NumSubElts, Index)) {
+ Mask, NumSrcElts, NumSubElts, Index)) {
SubTy = FixedVectorType::get(Ty->getElementType(), NumSubElts);
return TTI::SK_InsertSubvector;
}
- if (ShuffleVectorInst::isSelectMask(Mask))
+ if (ShuffleVectorInst::isSelectMask(Mask, NumSrcElts))
return TTI::SK_Select;
- if (ShuffleVectorInst::isTransposeMask(Mask))
+ if (ShuffleVectorInst::isTransposeMask(Mask, NumSrcElts))
return TTI::SK_Transpose;
- if (ShuffleVectorInst::isSpliceMask(Mask, Index))
+ if (ShuffleVectorInst::isSpliceMask(Mask, NumSrcElts, Index))
return TTI::SK_Splice;
break;
}
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index a0c8406..c85727c 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -2068,13 +2068,14 @@ public:
/// Return true if this shuffle mask chooses elements from exactly one source
/// vector.
/// Example: <7,5,undef,7>
- /// This assumes that vector operands are the same length as the mask.
- static bool isSingleSourceMask(ArrayRef<int> Mask);
- static bool isSingleSourceMask(const Constant *Mask) {
+ /// This assumes that vector operands (of length \p NumSrcElts) are the same
+ /// length as the mask.
+ static bool isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isSingleSourceMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isSingleSourceMask(MaskAsInts);
+ return isSingleSourceMask(MaskAsInts, NumSrcElts);
}
/// Return true if this shuffle chooses elements from exactly one source
@@ -2082,7 +2083,8 @@ public:
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
/// TODO: Optionally allow length-changing shuffles.
bool isSingleSource() const {
- return !changesLength() && isSingleSourceMask(ShuffleMask);
+ return !changesLength() &&
+ isSingleSourceMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle mask chooses elements from exactly one source
@@ -2090,8 +2092,8 @@ public:
/// necessarily a no-op because it may change the number of elements from its
/// input vectors or it may provide demanded bits knowledge via undef lanes.
/// Example: <undef,undef,2,3>
- static bool isIdentityMask(ArrayRef<int> Mask);
- static bool isIdentityMask(const Constant *Mask) {
+ static bool isIdentityMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isIdentityMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
// Not possible to express a shuffle mask for a scalable vector for this
@@ -2101,7 +2103,7 @@ public:
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isIdentityMask(MaskAsInts);
+ return isIdentityMask(MaskAsInts, NumSrcElts);
}
/// Return true if this shuffle chooses elements from exactly one source
@@ -2114,7 +2116,7 @@ public:
if (isa<ScalableVectorType>(getType()))
return false;
- return !changesLength() && isIdentityMask(ShuffleMask);
+ return !changesLength() && isIdentityMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle lengthens exactly one source vector with
@@ -2138,12 +2140,12 @@ public:
/// In that case, the shuffle is better classified as an identity shuffle.
/// This assumes that vector operands are the same length as the mask
/// (a length-changing shuffle can never be equivalent to a vector select).
- static bool isSelectMask(ArrayRef<int> Mask);
- static bool isSelectMask(const Constant *Mask) {
+ static bool isSelectMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isSelectMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isSelectMask(MaskAsInts);
+ return isSelectMask(MaskAsInts, NumSrcElts);
}
/// Return true if this shuffle chooses elements from its source vectors
@@ -2155,19 +2157,20 @@ public:
/// In that case, the shuffle is better classified as an identity shuffle.
/// TODO: Optionally allow length-changing shuffles.
bool isSelect() const {
- return !changesLength() && isSelectMask(ShuffleMask);
+ return !changesLength() && isSelectMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle mask swaps the order of elements from exactly
/// one source vector.
/// Example: <7,6,undef,4>
- /// This assumes that vector operands are the same length as the mask.
- static bool isReverseMask(ArrayRef<int> Mask);
- static bool isReverseMask(const Constant *Mask) {
+ /// This assumes that vector operands (of length \p NumSrcElts) are the same
+ /// length as the mask.
+ static bool isReverseMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isReverseMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isReverseMask(MaskAsInts);
+ return isReverseMask(MaskAsInts, NumSrcElts);
}
/// Return true if this shuffle swaps the order of elements from exactly
@@ -2175,19 +2178,20 @@ public:
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
/// TODO: Optionally allow length-changing shuffles.
bool isReverse() const {
- return !changesLength() && isReverseMask(ShuffleMask);
+ return !changesLength() && isReverseMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle mask chooses all elements with the same value
/// as the first element of exactly one source vector.
/// Example: <4,undef,undef,4>
- /// This assumes that vector operands are the same length as the mask.
- static bool isZeroEltSplatMask(ArrayRef<int> Mask);
- static bool isZeroEltSplatMask(const Constant *Mask) {
+ /// This assumes that vector operands (of length \p NumSrcElts) are the same
+ /// length as the mask.
+ static bool isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isZeroEltSplatMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isZeroEltSplatMask(MaskAsInts);
+ return isZeroEltSplatMask(MaskAsInts, NumSrcElts);
}
/// Return true if all elements of this shuffle are the same value as the
@@ -2197,7 +2201,8 @@ public:
/// TODO: Optionally allow length-changing shuffles.
/// TODO: Optionally allow splats from other elements.
bool isZeroEltSplat() const {
- return !changesLength() && isZeroEltSplatMask(ShuffleMask);
+ return !changesLength() &&
+ isZeroEltSplatMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle mask is a transpose mask.
@@ -2232,12 +2237,12 @@ public:
/// ; Transposed matrix
/// t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 >
/// t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 >
- static bool isTransposeMask(ArrayRef<int> Mask);
- static bool isTransposeMask(const Constant *Mask) {
+ static bool isTransposeMask(ArrayRef<int> Mask, int NumSrcElts);
+ static bool isTransposeMask(const Constant *Mask, int NumSrcElts) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isTransposeMask(MaskAsInts);
+ return isTransposeMask(MaskAsInts, NumSrcElts);
}
/// Return true if this shuffle transposes the elements of its inputs without
@@ -2246,19 +2251,21 @@ public:
/// exact specification.
/// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
bool isTranspose() const {
- return !changesLength() && isTransposeMask(ShuffleMask);
+ return !changesLength() && isTransposeMask(ShuffleMask, ShuffleMask.size());
}
/// Return true if this shuffle mask is a splice mask, concatenating the two
/// inputs together and then extracts an original width vector starting from
/// the splice index.
/// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
- static bool isSpliceMask(ArrayRef<int> Mask, int &Index);
- static bool isSpliceMask(const Constant *Mask, int &Index) {
+ /// This assumes that vector operands (of length \p NumSrcElts) are the same
+ /// length as the mask.
+ static bool isSpliceMask(ArrayRef<int> Mask, int NumSrcElts, int &Index);
+ static bool isSpliceMask(const Constant *Mask, int NumSrcElts, int &Index) {
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
SmallVector<int, 16> MaskAsInts;
getShuffleMask(Mask, MaskAsInts);
- return isSpliceMask(MaskAsInts, Index);
+ return isSpliceMask(MaskAsInts, NumSrcElts, Index);
}
/// Return true if this shuffle splices two inputs without changing the length
@@ -2266,7 +2273,8 @@ public:
/// then extracts an original width vector starting from the splice index.
/// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
bool isSplice(int &Index) const {
- return !changesLength() && isSpliceMask(ShuffleMask, Index);
+ return !changesLength() &&
+ isSpliceMask(ShuffleMask, ShuffleMask.size(), Index);
}
/// Return true if this shuffle mask is an extract subvector mask.
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 0d34ebb..c3a3704 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24119,7 +24119,7 @@ static SDValue foldExtractSubvectorFromShuffleVector(SDNode *N,
// Profitability check: only deal with extractions from the first subvector
// unless the mask becomes an identity mask.
- if (!ShuffleVectorInst::isIdentityMask(NewMask) ||
+ if (!ShuffleVectorInst::isIdentityMask(NewMask, NewMask.size()) ||
any_of(NewMask, [](int M) { return M < 0; }))
for (auto &DemandedSubvector : DemandedSubvectors)
if (DemandedSubvector.second != 0)
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index b76e355..ece3b58 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -2136,10 +2136,10 @@ static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
return UsesLHS || UsesRHS;
}
-bool ShuffleVectorInst::isSingleSourceMask(ArrayRef<int> Mask) {
+bool ShuffleVectorInst::isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts) {
// We don't have vector operand size information, so assume operands are the
// same size as the mask.
- return isSingleSourceMaskImpl(Mask, Mask.size());
+ return isSingleSourceMaskImpl(Mask, NumSrcElts);
}
static bool isIdentityMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
@@ -2154,65 +2154,75 @@ static bool isIdentityMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
return true;
}
-bool ShuffleVectorInst::isIdentityMask(ArrayRef<int> Mask) {
+bool ShuffleVectorInst::isIdentityMask(ArrayRef<int> Mask, int NumSrcElts) {
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
+ return false;
// We don't have vector operand size information, so assume operands are the
// same size as the mask.
- return isIdentityMaskImpl(Mask, Mask.size());
+ return isIdentityMaskImpl(Mask, NumSrcElts);
}
-bool ShuffleVectorInst::isReverseMask(ArrayRef<int> Mask) {
- if (!isSingleSourceMask(Mask))
+bool ShuffleVectorInst::isReverseMask(ArrayRef<int> Mask, int NumSrcElts) {
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
+ return false;
+ if (!isSingleSourceMask(Mask, NumSrcElts))
return false;
// The number of elements in the mask must be at least 2.
- int NumElts = Mask.size();
- if (NumElts < 2)
+ if (NumSrcElts < 2)
return false;
- for (int i = 0; i < NumElts; ++i) {
- if (Mask[i] == -1)
+ for (int I = 0, E = Mask.size(); I < E; ++I) {
+ if (Mask[I] == -1)
continue;
- if (Mask[i] != (NumElts - 1 - i) && Mask[i] != (NumElts + NumElts - 1 - i))
+ if (Mask[I] != (NumSrcElts - 1 - I) &&
+ Mask[I] != (NumSrcElts + NumSrcElts - 1 - I))
return false;
}
return true;
}
-bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef<int> Mask) {
- if (!isSingleSourceMask(Mask))
+bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts) {
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
return false;
- for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) {
- if (Mask[i] == -1)
+ if (!isSingleSourceMask(Mask, NumSrcElts))
+ return false;
+ for (int I = 0, E = Mask.size(); I < E; ++I) {
+ if (Mask[I] == -1)
continue;
- if (Mask[i] != 0 && Mask[i] != NumElts)
+ if (Mask[I] != 0 && Mask[I] != NumSrcElts)
return false;
}
return true;
}
-bool ShuffleVectorInst::isSelectMask(ArrayRef<int> Mask) {
+bool ShuffleVectorInst::isSelectMask(ArrayRef<int> Mask, int NumSrcElts) {
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
+ return false;
// Select is differentiated from identity. It requires using both sources.
- if (isSingleSourceMask(Mask))
+ if (isSingleSourceMask(Mask, NumSrcElts))
return false;
- for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) {
- if (Mask[i] == -1)
+ for (int I = 0, E = Mask.size(); I < E; ++I) {
+ if (Mask[I] == -1)
continue;
- if (Mask[i] != i && Mask[i] != (NumElts + i))
+ if (Mask[I] != I && Mask[I] != (NumSrcElts + I))
return false;
}
return true;
}
-bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask) {
+bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask, int NumSrcElts) {
// Example masks that will return true:
// v1 = <a, b, c, d>
// v2 = <e, f, g, h>
// trn1 = shufflevector v1, v2 <0, 4, 2, 6> = <a, e, c, g>
// trn2 = shufflevector v1, v2 <1, 5, 3, 7> = <b, f, d, h>
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
+ return false;
// 1. The number of elements in the mask must be a power-of-2 and at least 2.
- int NumElts = Mask.size();
- if (NumElts < 2 || !isPowerOf2_32(NumElts))
+ int Sz = Mask.size();
+ if (Sz < 2 || !isPowerOf2_32(Sz))
return false;
// 2. The first element of the mask must be either a 0 or a 1.
@@ -2221,23 +2231,26 @@ bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask) {
// 3. The difference between the first 2 elements must be equal to the
// number of elements in the mask.
- if ((Mask[1] - Mask[0]) != NumElts)
+ if ((Mask[1] - Mask[0]) != NumSrcElts)
return false;
// 4. The difference between consecutive even-numbered and odd-numbered
// elements must be equal to 2.
- for (int i = 2; i < NumElts; ++i) {
- int MaskEltVal = Mask[i];
+ for (int I = 2; I < Sz; ++I) {
+ int MaskEltVal = Mask[I];
if (MaskEltVal == -1)
return false;
- int MaskEltPrevVal = Mask[i - 2];
+ int MaskEltPrevVal = Mask[I - 2];
if (MaskEltVal - MaskEltPrevVal != 2)
return false;
}
return true;
}
-bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int &Index) {
+bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int NumSrcElts,
+ int &Index) {
+ if (Mask.size() != static_cast<unsigned>(NumSrcElts))
+ return false;
// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
int StartIndex = -1;
for (int I = 0, E = Mask.size(); I != E; ++I) {
@@ -2248,7 +2261,7 @@ bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int &Index) {
if (StartIndex == -1) {
// Don't support a StartIndex that begins in the second input, or if the
// first non-undef index would access below the StartIndex.
- if (MaskEltVal < I || E <= (MaskEltVal - I))
+ if (MaskEltVal < I || NumSrcElts <= (MaskEltVal - I))
return false;
StartIndex = MaskEltVal - I;
@@ -2543,7 +2556,7 @@ bool ShuffleVectorInst::isOneUseSingleSourceMask(int VF) const {
// case.
if (isa<ScalableVectorType>(getType()))
return false;
- if (!isSingleSourceMask(ShuffleMask))
+ if (!isSingleSourceMask(ShuffleMask, VF))
return false;
return isOneUseSingleSourceMask(ShuffleMask, VF);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index f887cf5..cba59c3 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -11802,7 +11802,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
if (((VT.getVectorNumElements() == 8 && VT.getScalarSizeInBits() == 16) ||
(VT.getVectorNumElements() == 16 && VT.getScalarSizeInBits() == 8)) &&
- ShuffleVectorInst::isReverseMask(ShuffleMask)) {
+ ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size())) {
SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1);
return DAG.getNode(AArch64ISD::EXT, dl, VT, Rev, Rev,
DAG.getConstant(8, dl, MVT::i32));
@@ -25946,7 +25946,8 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
unsigned MinSVESize = Subtarget->getMinSVEVectorSizeInBits();
unsigned MaxSVESize = Subtarget->getMaxSVEVectorSizeInBits();
if (MinSVESize == MaxSVESize && MaxSVESize == VT.getSizeInBits()) {
- if (ShuffleVectorInst::isReverseMask(ShuffleMask) && Op2.isUndef()) {
+ if (ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size()) &&
+ Op2.isUndef()) {
Op = DAG.getNode(ISD::VECTOR_REVERSE, DL, ContainerVT, Op1);
return convertFromScalableVector(DAG, VT, Op);
}
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 846094e..227346c 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -8375,7 +8375,7 @@ bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
unsigned EltSize = VT.getScalarSizeInBits();
if (EltSize >= 32 ||
ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
- ShuffleVectorInst::isIdentityMask(M) ||
+ ShuffleVectorInst::isIdentityMask(M, M.size()) ||
isVREVMask(M, VT, 64) ||
isVREVMask(M, VT, 32) ||
isVREVMask(M, VT, 16))
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 5e3975d..80691a7 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -4333,7 +4333,8 @@ static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN,
assert(VT.getVectorElementType() == MVT::i1);
- if (!ShuffleVectorInst::isReverseMask(SVN->getMask()) ||
+ if (!ShuffleVectorInst::isReverseMask(SVN->getMask(),
+ SVN->getMask().size()) ||
!SVN->getOperand(1).isUndef())
return SDValue();
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 2f510a2..e947d4d 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -322,7 +322,10 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
LT.second.getVectorElementType().getSizeInBits() ==
Tp->getElementType()->getPrimitiveSizeInBits() &&
LT.second.getVectorNumElements() <
- cast<FixedVectorType>(Tp)->getNumElements()) {
+ cast<FixedVectorType>(Tp)->getNumElements() &&
+ divideCeil(Mask.size(),
+ cast<FixedVectorType>(Tp)->getNumElements()) ==
+ static_cast<unsigned>(*LT.first.getValue())) {
unsigned NumRegs = *LT.first.getValue();
unsigned VF = cast<FixedVectorType>(Tp)->getNumElements();
unsigned SubVF = PowerOf2Ceil(VF / NumRegs);
@@ -498,7 +501,7 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
InstructionCost Cost = MemCost;
for (unsigned Index : Indices) {
FixedVectorType *SubVecTy =
- FixedVectorType::get(FVTy->getElementType(), VF);
+ FixedVectorType::get(FVTy->getElementType(), VF * Factor);
auto Mask = createStrideMask(Index, Factor, VF);
InstructionCost ShuffleCost =
getShuffleCost(TTI::ShuffleKind::SK_PermuteSingleSrc, SubVecTy, Mask,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index d838d1f..20c6431 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1635,7 +1635,7 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs, []() {},
[this, SingleOpTy, CostKind, &PrevSrcReg, &PrevRegMask,
&Cost](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
- if (!ShuffleVectorInst::isIdentityMask(RegMask)) {
+ if (!ShuffleVectorInst::isIdentityMask(RegMask, RegMask.size())) {
// Check if the previous register can be just copied to the next
// one.
if (PrevRegMask.empty() || PrevSrcReg != SrcReg ||
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 69d7593..c8b58c5 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -2122,8 +2122,8 @@ static Instruction *foldSelectShuffleOfSelectShuffle(ShuffleVectorInst &Shuf) {
NewMask[i] = Mask[i] < (signed)NumElts ? Mask[i] : Mask1[i];
// A select mask with undef elements might look like an identity mask.
- assert((ShuffleVectorInst::isSelectMask(NewMask) ||
- ShuffleVectorInst::isIdentityMask(NewMask)) &&
+ assert((ShuffleVectorInst::isSelectMask(NewMask, NumElts) ||
+ ShuffleVectorInst::isIdentityMask(NewMask, NumElts)) &&
"Unexpected shuffle mask");
return new ShuffleVectorInst(X, Y, NewMask);
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 181be11..49e9f5e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3812,7 +3812,7 @@ static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask) {
inversePermutation(Order, MaskOrder);
}
reorderReuses(MaskOrder, Mask);
- if (ShuffleVectorInst::isIdentityMask(MaskOrder)) {
+ if (ShuffleVectorInst::isIdentityMask(MaskOrder, MaskOrder.size())) {
Order.clear();
return;
}
@@ -4333,7 +4333,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
static bool isRepeatedNonIdentityClusteredMask(ArrayRef<int> Mask,
unsigned Sz) {
ArrayRef<int> FirstCluster = Mask.slice(0, Sz);
- if (ShuffleVectorInst::isIdentityMask(FirstCluster))
+ if (ShuffleVectorInst::isIdentityMask(FirstCluster, Sz))
return false;
for (unsigned I = Sz, E = Mask.size(); I < E; I += Sz) {
ArrayRef<int> Cluster = Mask.slice(I, Sz);
@@ -6646,9 +6646,24 @@ protected:
bool IsStrict) {
int Limit = Mask.size();
int VF = VecTy->getNumElements();
- return (VF == Limit || !IsStrict) &&
- all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) &&
- ShuffleVectorInst::isIdentityMask(Mask);
+ int Index = -1;
+ if (VF == Limit && ShuffleVectorInst::isIdentityMask(Mask, Limit))
+ return true;
+ if (!IsStrict) {
+ // Consider extract subvector starting from index 0.
+ if (ShuffleVectorInst::isExtractSubvectorMask(Mask, VF, Index) &&
+ Index == 0)
+ return true;
+ // All VF-size submasks are identity (e.g.
+ // <poison,poison,poison,poison,0,1,2,poison,poison,1,2,3> etc. for VF 4).
+ if (Limit % VF == 0 && all_of(seq<int>(0, Limit / VF), [=](int Idx) {
+ ArrayRef<int> Slice = Mask.slice(Idx * VF, VF);
+ return all_of(Slice, [](int I) { return I == PoisonMaskElem; }) ||
+ ShuffleVectorInst::isIdentityMask(Slice, VF);
+ }))
+ return true;
+ }
+ return false;
}
/// Tries to combine 2 different masks into single one.
@@ -6718,7 +6733,8 @@ protected:
if (isIdentityMask(Mask, SVTy, /*IsStrict=*/false)) {
if (!IdentityOp || !SinglePermute ||
(isIdentityMask(Mask, SVTy, /*IsStrict=*/true) &&
- !ShuffleVectorInst::isZeroEltSplatMask(IdentityMask))) {
+ !ShuffleVectorInst::isZeroEltSplatMask(IdentityMask,
+ IdentityMask.size()))) {
IdentityOp = SV;
// Store current mask in the IdentityMask so later we did not lost
// this info if IdentityOp is selected as the best candidate for the
@@ -6788,7 +6804,7 @@ protected:
}
if (auto *OpTy = dyn_cast<FixedVectorType>(Op->getType());
!OpTy || !isIdentityMask(Mask, OpTy, SinglePermute) ||
- ShuffleVectorInst::isZeroEltSplatMask(Mask)) {
+ ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size())) {
if (IdentityOp) {
V = IdentityOp;
assert(Mask.size() == IdentityMask.size() &&
@@ -6804,7 +6820,7 @@ protected:
/*IsStrict=*/true) ||
(Shuffle && Mask.size() == Shuffle->getShuffleMask().size() &&
Shuffle->isZeroEltSplat() &&
- ShuffleVectorInst::isZeroEltSplatMask(Mask)));
+ ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size())));
}
V = Op;
return false;
@@ -6909,11 +6925,9 @@ protected:
CombinedMask1[I] = CombinedMask2[I] + (Op1 == Op2 ? 0 : VF);
}
}
- const int Limit = CombinedMask1.size() * 2;
- if (Op1 == Op2 && Limit == 2 * VF &&
- all_of(CombinedMask1, [=](int Idx) { return Idx < Limit; }) &&
- (ShuffleVectorInst::isIdentityMask(CombinedMask1) ||
- (ShuffleVectorInst::isZeroEltSplatMask(CombinedMask1) &&
+ if (Op1 == Op2 &&
+ (ShuffleVectorInst::isIdentityMask(CombinedMask1, VF) ||
+ (ShuffleVectorInst::isZeroEltSplatMask(CombinedMask1, VF) &&
isa<ShuffleVectorInst>(Op1) &&
cast<ShuffleVectorInst>(Op1)->getShuffleMask() ==
ArrayRef(CombinedMask1))))
@@ -7183,11 +7197,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
const TargetTransformInfo &TTI;
static bool isEmptyOrIdentity(ArrayRef<int> Mask, unsigned VF) {
- int Limit = 2 * VF;
return Mask.empty() ||
(VF == Mask.size() &&
- all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) &&
- ShuffleVectorInst::isIdentityMask(Mask));
+ ShuffleVectorInst::isIdentityMask(Mask, VF));
}
public:
@@ -7445,9 +7457,7 @@ public:
::addMask(CommonMask, ExtMask, /*ExtendingManyInputs=*/true);
if (CommonMask.empty())
return Cost;
- int Limit = CommonMask.size() * 2;
- if (all_of(CommonMask, [=](int Idx) { return Idx < Limit; }) &&
- ShuffleVectorInst::isIdentityMask(CommonMask))
+ if (ShuffleVectorInst::isIdentityMask(CommonMask, CommonMask.size()))
return Cost;
return Cost +
createShuffle(InVectors.front(),
@@ -7624,7 +7634,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
}
if (NeedToShuffleReuses)
::addMask(Mask, E->ReuseShuffleIndices);
- if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask))
+ if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
CommonCost =
TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, FinalVecTy, Mask);
assert((E->State == TreeEntry::Vectorize ||
@@ -7975,7 +7985,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
} else {
SmallVector<int> Mask;
inversePermutation(OpTE->ReorderIndices, Mask);
- if (ShuffleVectorInst::isReverseMask(Mask))
+ if (ShuffleVectorInst::isReverseMask(Mask, Mask.size()))
CCH = TTI::CastContextHint::Reversed;
}
}
@@ -8834,9 +8844,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
unsigned VecVF = TE->getVectorFactor();
if (VF != VecVF &&
(any_of(Mask, [VF](int Idx) { return Idx >= static_cast<int>(VF); }) ||
- (all_of(Mask,
- [VF](int Idx) { return Idx < 2 * static_cast<int>(VF); }) &&
- !ShuffleVectorInst::isIdentityMask(Mask)))) {
+ !ShuffleVectorInst::isIdentityMask(Mask, VF))) {
SmallVector<int> OrigMask(VecVF, PoisonMaskElem);
std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)),
OrigMask.begin());
@@ -8865,9 +8873,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
assert((TEs.size() == 1 || TEs.size() == 2) &&
"Expected exactly 1 or 2 tree entries.");
if (TEs.size() == 1) {
- int Limit = 2 * Mask.size();
- if (!all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) ||
- !ShuffleVectorInst::isIdentityMask(Mask)) {
+ if (!ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) {
InstructionCost C =
TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, FTy, Mask);
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
@@ -9643,7 +9649,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
return V1;
unsigned VF = Mask.size();
unsigned LocalVF = cast<FixedVectorType>(V1->getType())->getNumElements();
- if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask))
+ if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask, VF))
return V1;
Value *Vec = Builder.CreateShuffleVector(V1, Mask);
if (auto *I = dyn_cast<Instruction>(Vec)) {
@@ -10056,9 +10062,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
return false;
unsigned I =
*find_if_not(Mask, [](int Idx) { return Idx == PoisonMaskElem; });
- int Sz = Mask.size();
- if (all_of(Mask, [Sz](int Idx) { return Idx < 2 * Sz; }) &&
- ShuffleVectorInst::isIdentityMask(Mask))
+ if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
std::iota(Mask.begin(), Mask.end(), 0);
else
std::fill(Mask.begin(), Mask.end(), I);
@@ -10302,11 +10306,11 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
(ExtractShuffle.value_or(TTI::SK_PermuteTwoSrc) ==
TTI::SK_PermuteSingleSrc &&
none_of(ExtractMask, [&](int I) { return I >= EMSz; }) &&
- ShuffleVectorInst::isIdentityMask(ExtractMask)) ||
+ ShuffleVectorInst::isIdentityMask(ExtractMask, EMSz)) ||
(GatherShuffle.value_or(TTI::SK_PermuteTwoSrc) ==
TTI::SK_PermuteSingleSrc &&
none_of(Mask, [&](int I) { return I >= MSz; }) &&
- ShuffleVectorInst::isIdentityMask(Mask));
+ ShuffleVectorInst::isIdentityMask(Mask, MSz));
bool EnoughConstsForShuffle =
IsSingleShuffle &&
(none_of(GatheredScalars,
@@ -10599,7 +10603,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
if ((!IsIdentity || Offset != 0 || !IsFirstUndef.all()) &&
NumElts != NumScalars) {
if (IsFirstUndef.all()) {
- if (!ShuffleVectorInst::isIdentityMask(InsertMask)) {
+ if (!ShuffleVectorInst::isIdentityMask(InsertMask, NumElts)) {
SmallBitVector IsFirstPoison =
isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
if (!IsFirstPoison.all()) {
@@ -11422,7 +11426,7 @@ Value *BoUpSLP::vectorizeTree(
// non-resizing mask.
if (Mask.size() != cast<FixedVectorType>(Vals.front()->getType())
->getNumElements() ||
- !ShuffleVectorInst::isIdentityMask(Mask))
+ !ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
return CreateShuffle(Vals.front(), nullptr, Mask);
return Vals.front();
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index d6196c8..99f03cd 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -461,44 +461,21 @@ exit:
define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
; CHECK-LABEL: @combine_load_factor2_i64(
; CHECK-NEXT: entry:
-; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC1]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0
-; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP6]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
; CHECK-NEXT: [[X0:%.*]] = load i64, ptr [[Q0]], align 4
; CHECK-NEXT: [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
; CHECK-NEXT: [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
; CHECK-NEXT: [[X1:%.*]] = load i64, ptr [[Q1]], align 4
; CHECK-NEXT: [[RES:%.*]] = add i64 [[X0]], [[X1]]
-; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]]
+; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[I]]
; CHECK-NEXT: store i64 [[RES]], ptr [[DST]], align 4
; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index 637c692..20b8529 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -1024,71 +1024,141 @@ TEST(InstructionsTest, ShuffleMaskQueries) {
Constant *C7 = ConstantInt::get(Int32Ty, 7);
Constant *Identity = ConstantVector::get({C0, CU, C2, C3, C4});
- EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(Identity));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Identity)); // identity is distinguished from select
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Identity));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(Identity)); // identity is always single source
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Identity));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Identity));
+ EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(
+ Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
+ Identity,
+ cast<FixedVectorType>(Identity->getType())
+ ->getNumElements())); // identity is distinguished from select
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
+ Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ Identity, cast<FixedVectorType>(Identity->getType())
+ ->getNumElements())); // identity is always single source
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
+ Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
+ Identity, cast<FixedVectorType>(Identity->getType())->getNumElements()));
Constant *Select = ConstantVector::get({CU, C1, C5});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Select));
- EXPECT_TRUE(ShuffleVectorInst::isSelectMask(Select));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Select));
- EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(Select));
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Select));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Select));
-
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isSelectMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
+ Select, cast<FixedVectorType>(Select->getType())->getNumElements()));
+
Constant *Reverse = ConstantVector::get({C3, C2, C1, CU});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Reverse));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Reverse));
- EXPECT_TRUE(ShuffleVectorInst::isReverseMask(Reverse));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(Reverse)); // reverse is always single source
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Reverse));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Reverse));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isReverseMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())
+ ->getNumElements())); // reverse is always single source
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
+ Reverse, cast<FixedVectorType>(Reverse->getType())->getNumElements()));
Constant *SingleSource = ConstantVector::get({C2, C2, C0, CU});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(SingleSource));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(SingleSource));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(SingleSource));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(SingleSource));
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(SingleSource));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(SingleSource));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
+ SingleSource,
+ cast<FixedVectorType>(SingleSource->getType())->getNumElements()));
Constant *ZeroEltSplat = ConstantVector::get({C0, C0, CU, C0});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(ZeroEltSplat));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(ZeroEltSplat));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(ZeroEltSplat));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ZeroEltSplat)); // 0-splat is always single source
- EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ZeroEltSplat));
- EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(ZeroEltSplat));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
+ ZeroEltSplat,
+ cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
+ ZeroEltSplat,
+ cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
+ ZeroEltSplat,
+ cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ ZeroEltSplat, cast<FixedVectorType>(ZeroEltSplat->getType())
+ ->getNumElements())); // 0-splat is always single source
+ EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(
+ ZeroEltSplat,
+ cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(
+ ZeroEltSplat,
+ cast<FixedVectorType>(ZeroEltSplat->getType())->getNumElements()));
Constant *Transpose = ConstantVector::get({C0, C4, C2, C6});
- EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Transpose));
- EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Transpose));
- EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Transpose));
- EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(Transpose));
- EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Transpose));
- EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(Transpose));
+ EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSelectMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isReverseMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
+ EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
+ EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(
+ Transpose,
+ cast<FixedVectorType>(Transpose->getType())->getNumElements()));
// More tests to make sure the logic is/stays correct...
- EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(ConstantVector::get({CU, C1, CU, C3})));
- EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(ConstantVector::get({C4, CU, C6, CU})));
-
- EXPECT_TRUE(ShuffleVectorInst::isSelectMask(ConstantVector::get({C4, C1, C6, CU})));
- EXPECT_TRUE(ShuffleVectorInst::isSelectMask(ConstantVector::get({CU, C1, C6, C3})));
-
- EXPECT_TRUE(ShuffleVectorInst::isReverseMask(ConstantVector::get({C7, C6, CU, C4})));
- EXPECT_TRUE(ShuffleVectorInst::isReverseMask(ConstantVector::get({C3, CU, C1, CU})));
-
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ConstantVector::get({C7, C5, CU, C7})));
- EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ConstantVector::get({C3, C0, CU, C3})));
-
- EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ConstantVector::get({C4, CU, CU, C4})));
- EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ConstantVector::get({CU, C0, CU, C0})));
-
- EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C5, C3, C7})));
- EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C3})));
+ EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(
+ ConstantVector::get({CU, C1, CU, C3}), 4));
+ EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(
+ ConstantVector::get({C4, CU, C6, CU}), 4));
+
+ EXPECT_TRUE(ShuffleVectorInst::isSelectMask(
+ ConstantVector::get({C4, C1, C6, CU}), 4));
+ EXPECT_TRUE(ShuffleVectorInst::isSelectMask(
+ ConstantVector::get({CU, C1, C6, C3}), 4));
+
+ EXPECT_TRUE(ShuffleVectorInst::isReverseMask(
+ ConstantVector::get({C7, C6, CU, C4}), 4));
+ EXPECT_TRUE(ShuffleVectorInst::isReverseMask(
+ ConstantVector::get({C3, CU, C1, CU}), 4));
+
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ ConstantVector::get({C7, C5, CU, C7}), 4));
+ EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(
+ ConstantVector::get({C3, C0, CU, C3}), 4));
+
+ EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(
+ ConstantVector::get({C4, CU, CU, C4}), 4));
+ EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(
+ ConstantVector::get({CU, C0, CU, C0}), 4));
+
+ EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(
+ ConstantVector::get({C1, C5, C3, C7}), 4));
+ EXPECT_TRUE(
+ ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C3}), 2));
// Nothing special about the values here - just re-using inputs to reduce code.
Constant *V0 = ConstantVector::get({C0, C1, C2, C3});
diff --git a/llvm/unittests/IR/ShuffleVectorInstTest.cpp b/llvm/unittests/IR/ShuffleVectorInstTest.cpp
index ba1b980..5d840ec 100644
--- a/llvm/unittests/IR/ShuffleVectorInstTest.cpp
+++ b/llvm/unittests/IR/ShuffleVectorInstTest.cpp
@@ -14,51 +14,84 @@ using namespace llvm;
namespace {
TEST(ShuffleVectorInst, isIdentityMask) {
- ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}));
- ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}));
- ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}));
+ ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 4));
+ ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 5));
+ ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 4));
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 4}));
- ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, -1, 2, 4}));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 4));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 3));
+
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 5));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 6));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 5));
+
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 4}, 4));
+ ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, -1, 2, 4}, 4));
}
TEST(ShuffleVectorInst, isSelectMask) {
- ASSERT_TRUE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}));
+ ASSERT_TRUE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 4));
+
+ ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 5));
- ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 1, 2, 3}));
+ ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 1, 2, 3}, 4));
}
TEST(ShuffleVectorInst, isReverseMask) {
- ASSERT_TRUE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}));
- ASSERT_TRUE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}));
+ ASSERT_TRUE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 4));
+ ASSERT_TRUE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 4));
- ASSERT_FALSE(ShuffleVectorInst::isReverseMask({4, 3, 2, 1}));
+ ASSERT_FALSE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 5));
+ ASSERT_FALSE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 5));
+
+ ASSERT_FALSE(ShuffleVectorInst::isReverseMask({4, 3, 2, 1}, 4));
}
TEST(ShuffleVectorInst, isZeroEltSplatMask) {
- ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}));
- ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}));
+ ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 4));
+ ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 4));
+
+ ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 5));
+ ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 5));
- ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({1, 1, 1, 1}));
+ ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({1, 1, 1, 1}, 4));
}
TEST(ShuffleVectorInst, isTransposeMask) {
- ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}));
- ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}));
+ ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 4));
+ ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 4));
- ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({2, 6, 4, 8}));
+ ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 3));
+ ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 5));
+ ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 5));
+
+ ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({2, 6, 4, 8}, 4));
}
TEST(ShuffleVectorInst, isSpliceMask) {
int Index;
- ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, Index));
+ ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 4, Index));
ASSERT_EQ(0, Index);
- ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, Index));
+ ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 7, Index));
ASSERT_EQ(1, Index);
- ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({4, 5, 6, 7}, Index));
+ ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 3, Index));
+ ASSERT_FALSE(
+ ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 6, Index));
+ ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 5, Index));
+ ASSERT_FALSE(
+ ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 8, Index));
+
+ ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({4, 5, 6, 7}, 4, Index));
}
TEST(ShuffleVectorInst, isExtractSubvectorMask) {
generated by cgit v1.1 at 2025-08-02 07:26:10 +0000