diff options
author | Florian Hahn <flo@fhahn.com> | 2024-06-19 20:17:00 +0100 |
---|---|---|
committer | Florian Hahn <flo@fhahn.com> | 2024-06-19 20:17:01 +0100 |
commit | c008647b3a60efdbb5499c3df2b3e403728c29da (patch) | |
tree | ef4b57012431458aa578ba1c41966b53d5571d1f /llvm | |
parent | 936bc9bb07f06b0fb036fb6f9d19f79aa3a12cf5 (diff) | |
download | llvm-c008647b3a60efdbb5499c3df2b3e403728c29da.zip llvm-c008647b3a60efdbb5499c3df2b3e403728c29da.tar.gz llvm-c008647b3a60efdbb5499c3df2b3e403728c29da.tar.bz2 |
[VPlan] Introduce isHeaderMask helper (NFCI).
Split off from https://github.com/llvm/llvm-project/pull/92555
and slightly generalized to more precisely check for a header mask.
Use it to replace manual checks in collectHeaderMasks.
Diffstat (limited to 'llvm')
-rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlan.cpp | 20 | ||||
-rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlan.h | 4 | ||||
-rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 40 | ||||
-rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 5 |
4 files changed, 65 insertions, 4 deletions
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 348a2be..0016fec 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1459,3 +1459,23 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, Plan.addSCEVExpansion(Expr, Expanded); return Expanded; } + +bool vputils::isHeaderMask(VPValue *V, VPlan &Plan) { + if (isa<VPActiveLaneMaskPHIRecipe>(V)) + return true; + + auto IsWideCanonicalIV = [](VPValue *A) { + return isa<VPWidenCanonicalIVRecipe>(A) || + (isa<VPWidenIntOrFpInductionRecipe>(A) && + cast<VPWidenIntOrFpInductionRecipe>(A)->isCanonical()); + }; + + VPValue *A, *B; + if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B)))) + return B == Plan.getTripCount() && + (match(A, m_ScalarIVSteps(m_CanonicalIV(), m_SpecificInt(1))) || + IsWideCanonicalIV(A)); + + return match(V, m_Binary<Instruction::ICmp>(m_VPValue(A), m_VPValue(B))) && + IsWideCanonicalIV(A) && B == Plan.getOrCreateBackedgeTakenCount(); +} diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 5bb88e4..fc25ed9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3665,6 +3665,10 @@ inline bool isUniformAfterVectorization(VPValue *VPV) { return VPI->isVectorToScalar(); return false; } + +/// Return true if \p V is a header mask in \p Plan. +bool isHeaderMask(VPValue *V, VPlan &Plan); + } // end namespace vputils } // end namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index d6b4acb..9cd7712 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -303,6 +303,46 @@ inline BinaryVPInstruction_match<Op0_t, Op1_t, VPInstruction::LogicalAnd> m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) { return m_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1); } + +struct VPCanonicalIVPHI_match { + bool match(const VPValue *V) { + auto *DefR = V->getDefiningRecipe(); + return DefR && match(DefR); + } + + bool match(const VPRecipeBase *R) { return isa<VPCanonicalIVPHIRecipe>(R); } +}; + +inline VPCanonicalIVPHI_match m_CanonicalIV() { + return VPCanonicalIVPHI_match(); +} + +template <typename Op0_t, typename Op1_t> struct VPScalarIVSteps_match { + Op0_t Op0; + Op1_t Op1; + + VPScalarIVSteps_match(Op0_t Op0, Op1_t Op1) : Op0(Op0), Op1(Op1) {} + + bool match(const VPValue *V) { + auto *DefR = V->getDefiningRecipe(); + return DefR && match(DefR); + } + + bool match(const VPRecipeBase *R) { + if (!isa<VPScalarIVStepsRecipe>(R)) + return false; + assert(R->getNumOperands() == 2 && + "VPScalarIVSteps must have exactly 2 operands"); + return Op0.match(R->getOperand(0)) && Op1.match(R->getOperand(1)); + } +}; + +template <typename Op0_t, typename Op1_t> +inline VPScalarIVSteps_match<Op0_t, Op1_t> m_ScalarIVSteps(const Op0_t &Op0, + const Op1_t &Op1) { + return VPScalarIVSteps_match<Op0_t, Op1_t>(Op0, Op1); +} + } // namespace VPlanPatternMatch } // namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 8ec67eb..e2b7b0c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1334,13 +1334,10 @@ static SmallVector<VPValue *> collectAllHeaderMasks(VPlan &Plan) { // Walk users of wide canonical IVs and collect to all compares of the form // (ICMP_ULE, WideCanonicalIV, backedge-taken-count). SmallVector<VPValue *> HeaderMasks; - VPValue *BTC = Plan.getOrCreateBackedgeTakenCount(); for (auto *Wide : WideCanonicalIVs) { for (VPUser *U : SmallVector<VPUser *>(Wide->users())) { auto *HeaderMask = dyn_cast<VPInstruction>(U); - if (!HeaderMask || HeaderMask->getOpcode() != Instruction::ICmp || - HeaderMask->getPredicate() != CmpInst::ICMP_ULE || - HeaderMask->getOperand(1) != BTC) + if (!HeaderMask || !vputils::isHeaderMask(HeaderMask, Plan)) continue; assert(HeaderMask->getOperand(0) == Wide && |