aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/InterleavedAccessPass.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/InterleavedAccessPass.cpp')
-rw-r--r--llvm/lib/CodeGen/InterleavedAccessPass.cpp199
1 files changed, 98 insertions, 101 deletions
diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
index d2b2edf..65565b9 100644
--- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
@@ -253,6 +253,21 @@ static bool isReInterleaveMask(ShuffleVectorInst *SVI, unsigned &Factor,
return false;
}
+static Value *getMaskOperand(IntrinsicInst *II) {
+ switch (II->getIntrinsicID()) {
+ default:
+ llvm_unreachable("Unexpected intrinsic");
+ case Intrinsic::vp_load:
+ return II->getOperand(1);
+ case Intrinsic::masked_load:
+ return II->getOperand(2);
+ case Intrinsic::vp_store:
+ return II->getOperand(2);
+ case Intrinsic::masked_store:
+ return II->getOperand(3);
+ }
+}
+
// Return the corresponded deinterleaved mask, or nullptr if there is no valid
// mask.
static Value *getMask(Value *WideMask, unsigned Factor,
@@ -268,17 +283,13 @@ bool InterleavedAccessImpl::lowerInterleavedLoad(
if (isa<ScalableVectorType>(Load->getType()))
return false;
- if (auto *LI = dyn_cast<LoadInst>(Load)) {
- if (!LI->isSimple())
- return false;
- } else if (auto *VPLoad = dyn_cast<VPIntrinsic>(Load)) {
- assert(VPLoad->getIntrinsicID() == Intrinsic::vp_load);
- // Require a constant mask.
- if (!isa<ConstantVector>(VPLoad->getMaskParam()))
- return false;
- } else {
- llvm_unreachable("unsupported load operation");
- }
+ auto *LI = dyn_cast<LoadInst>(Load);
+ auto *II = dyn_cast<IntrinsicInst>(Load);
+ if (!LI && !II)
+ return false;
+
+ if (LI && !LI->isSimple())
+ return false;
// Check if all users of this load are shufflevectors. If we encounter any
// users that are extractelement instructions or binary operators, we save
@@ -330,7 +341,7 @@ bool InterleavedAccessImpl::lowerInterleavedLoad(
// Holds the corresponding index for each DE-interleave shuffle.
SmallVector<unsigned, 4> Indices;
- Type *VecTy = FirstSVI->getType();
+ VectorType *VecTy = cast<VectorType>(FirstSVI->getType());
// Check if other shufflevectors are also DE-interleaved of the same type
// and factor as the first shufflevector.
@@ -368,13 +379,16 @@ bool InterleavedAccessImpl::lowerInterleavedLoad(
replaceBinOpShuffles(BinOpShuffles.getArrayRef(), Shuffles, Load);
Value *Mask = nullptr;
- if (auto *VPLoad = dyn_cast<VPIntrinsic>(Load)) {
- Mask = getMask(VPLoad->getMaskParam(), Factor, cast<VectorType>(VecTy));
+ if (LI) {
+ LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *Load << "\n");
+ } else {
+ // Check mask operand. Handle both all-true/false and interleaved mask.
+ Mask = getMask(getMaskOperand(II), Factor, VecTy);
if (!Mask)
return false;
- LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.load: " << *Load << "\n");
- } else {
- LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *Load << "\n");
+
+ LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.load or masked.load: "
+ << *Load << "\n");
}
// Try to create target specific intrinsics to replace the load and
@@ -491,18 +505,16 @@ bool InterleavedAccessImpl::tryReplaceExtracts(
bool InterleavedAccessImpl::lowerInterleavedStore(
Instruction *Store, SmallSetVector<Instruction *, 32> &DeadInsts) {
Value *StoredValue;
- if (auto *SI = dyn_cast<StoreInst>(Store)) {
+ auto *SI = dyn_cast<StoreInst>(Store);
+ auto *II = dyn_cast<IntrinsicInst>(Store);
+ if (SI) {
if (!SI->isSimple())
return false;
StoredValue = SI->getValueOperand();
- } else if (auto *VPStore = dyn_cast<VPIntrinsic>(Store)) {
- assert(VPStore->getIntrinsicID() == Intrinsic::vp_store);
- // Require a constant mask.
- if (!isa<ConstantVector>(VPStore->getMaskParam()))
- return false;
- StoredValue = VPStore->getArgOperand(0);
} else {
- llvm_unreachable("unsupported store operation");
+ assert(II->getIntrinsicID() == Intrinsic::vp_store ||
+ II->getIntrinsicID() == Intrinsic::masked_store);
+ StoredValue = II->getArgOperand(0);
}
auto *SVI = dyn_cast<ShuffleVectorInst>(StoredValue);
@@ -518,46 +530,26 @@ bool InterleavedAccessImpl::lowerInterleavedStore(
assert(NumStoredElements % Factor == 0 &&
"number of stored element should be a multiple of Factor");
- if (auto *VPStore = dyn_cast<VPIntrinsic>(Store)) {
+ Value *Mask = nullptr;
+ if (SI) {
+ LLVM_DEBUG(dbgs() << "IA: Found an interleaved store: " << *Store << "\n");
+ } else {
+ // Check mask operand. Handle both all-true/false and interleaved mask.
unsigned LaneMaskLen = NumStoredElements / Factor;
- Value *LaneMask = getMask(VPStore->getMaskParam(), Factor,
- ElementCount::getFixed(LaneMaskLen));
- if (!LaneMask)
+ Mask = getMask(getMaskOperand(II), Factor,
+ ElementCount::getFixed(LaneMaskLen));
+ if (!Mask)
return false;
- LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.store: " << *Store
- << "\n");
-
- IRBuilder<> Builder(VPStore);
- // We need to effectively de-interleave the shufflemask
- // because lowerInterleavedVPStore expects individual de-interleaved
- // values.
- SmallVector<Value *, 10> NewShuffles;
- SmallVector<int, 16> NewShuffleMask(LaneMaskLen);
- auto ShuffleMask = SVI->getShuffleMask();
-
- for (unsigned i = 0; i < Factor; i++) {
- for (unsigned j = 0; j < LaneMaskLen; j++)
- NewShuffleMask[j] = ShuffleMask[i + Factor * j];
-
- NewShuffles.push_back(Builder.CreateShuffleVector(
- SVI->getOperand(0), SVI->getOperand(1), NewShuffleMask));
- }
-
- // Try to create target specific intrinsics to replace the vp.store and
- // shuffle.
- if (!TLI->lowerInterleavedVPStore(VPStore, LaneMask, NewShuffles))
- // We already created new shuffles.
- return true;
- } else {
- LLVM_DEBUG(dbgs() << "IA: Found an interleaved store: " << *Store << "\n");
-
- // Try to create target specific intrinsics to replace the store and
- // shuffle.
- if (!TLI->lowerInterleavedStore(cast<StoreInst>(Store), SVI, Factor))
- return false;
+ LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.store or masked.store: "
+ << *Store << "\n");
}
+ // Try to create target specific intrinsics to replace the store and
+ // shuffle.
+ if (!TLI->lowerInterleavedStore(Store, Mask, SVI, Factor))
+ return false;
+
// Already have a new target specific interleaved store. Erase the old store.
DeadInsts.insert(Store);
DeadInsts.insert(SVI);
@@ -600,87 +592,90 @@ static Value *getMask(Value *WideMask, unsigned Factor,
bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
IntrinsicInst *DI, SmallSetVector<Instruction *, 32> &DeadInsts) {
- Value *LoadedVal = DI->getOperand(0);
- if (!LoadedVal->hasOneUse() || !isa<LoadInst, VPIntrinsic>(LoadedVal))
+ Instruction *LoadedVal = dyn_cast<Instruction>(DI->getOperand(0));
+ if (!LoadedVal || !LoadedVal->hasOneUse())
+ return false;
+
+ auto *LI = dyn_cast<LoadInst>(LoadedVal);
+ auto *II = dyn_cast<IntrinsicInst>(LoadedVal);
+ if (!LI && !II)
return false;
const unsigned Factor = getDeinterleaveIntrinsicFactor(DI->getIntrinsicID());
assert(Factor && "unexpected deinterleave intrinsic");
Value *Mask = nullptr;
- if (auto *VPLoad = dyn_cast<VPIntrinsic>(LoadedVal)) {
- if (VPLoad->getIntrinsicID() != Intrinsic::vp_load)
- return false;
- // Check mask operand. Handle both all-true/false and interleaved mask.
- Value *WideMask = VPLoad->getOperand(1);
- Mask = getMask(WideMask, Factor, getDeinterleavedVectorType(DI));
- if (!Mask)
- return false;
-
- LLVM_DEBUG(dbgs() << "IA: Found a vp.load with deinterleave intrinsic "
- << *DI << " and factor = " << Factor << "\n");
- } else {
- auto *LI = cast<LoadInst>(LoadedVal);
+ if (LI) {
if (!LI->isSimple())
return false;
LLVM_DEBUG(dbgs() << "IA: Found a load with deinterleave intrinsic " << *DI
<< " and factor = " << Factor << "\n");
+ } else {
+ assert(II);
+
+ // Check mask operand. Handle both all-true/false and interleaved mask.
+ Mask = getMask(getMaskOperand(II), Factor, getDeinterleavedVectorType(DI));
+ if (!Mask)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "IA: Found a vp.load or masked.load with deinterleave"
+ << " intrinsic " << *DI << " and factor = "
+ << Factor << "\n");
}
// Try and match this with target specific intrinsics.
- if (!TLI->lowerDeinterleaveIntrinsicToLoad(cast<Instruction>(LoadedVal), Mask,
- DI))
+ if (!TLI->lowerDeinterleaveIntrinsicToLoad(LoadedVal, Mask, DI))
return false;
DeadInsts.insert(DI);
// We now have a target-specific load, so delete the old one.
- DeadInsts.insert(cast<Instruction>(LoadedVal));
+ DeadInsts.insert(LoadedVal);
return true;
}
bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
- IntrinsicInst *II, SmallSetVector<Instruction *, 32> &DeadInsts) {
- if (!II->hasOneUse())
+ IntrinsicInst *IntII, SmallSetVector<Instruction *, 32> &DeadInsts) {
+ if (!IntII->hasOneUse())
+ return false;
+ Instruction *StoredBy = dyn_cast<Instruction>(IntII->user_back());
+ if (!StoredBy)
return false;
- Value *StoredBy = II->user_back();
- if (!isa<StoreInst, VPIntrinsic>(StoredBy))
+ auto *SI = dyn_cast<StoreInst>(StoredBy);
+ auto *II = dyn_cast<IntrinsicInst>(StoredBy);
+ if (!SI && !II)
return false;
- SmallVector<Value *, 8> InterleaveValues(II->args());
- const unsigned Factor = getInterleaveIntrinsicFactor(II->getIntrinsicID());
+ SmallVector<Value *, 8> InterleaveValues(IntII->args());
+ const unsigned Factor = getInterleaveIntrinsicFactor(IntII->getIntrinsicID());
assert(Factor && "unexpected interleave intrinsic");
Value *Mask = nullptr;
- if (auto *VPStore = dyn_cast<VPIntrinsic>(StoredBy)) {
- if (VPStore->getIntrinsicID() != Intrinsic::vp_store)
- return false;
-
- Value *WideMask = VPStore->getOperand(2);
- Mask = getMask(WideMask, Factor,
+ if (II) {
+ // Check mask operand. Handle both all-true/false and interleaved mask.
+ Mask = getMask(getMaskOperand(II), Factor,
cast<VectorType>(InterleaveValues[0]->getType()));
if (!Mask)
return false;
- LLVM_DEBUG(dbgs() << "IA: Found a vp.store with interleave intrinsic "
- << *II << " and factor = " << Factor << "\n");
+ LLVM_DEBUG(dbgs() << "IA: Found a vp.store or masked.store with interleave"
+ << " intrinsic " << *IntII << " and factor = "
+ << Factor << "\n");
} else {
- auto *SI = cast<StoreInst>(StoredBy);
if (!SI->isSimple())
return false;
- LLVM_DEBUG(dbgs() << "IA: Found a store with interleave intrinsic " << *II
- << " and factor = " << Factor << "\n");
+ LLVM_DEBUG(dbgs() << "IA: Found a store with interleave intrinsic "
+ << *IntII << " and factor = " << Factor << "\n");
}
// Try and match this with target specific intrinsics.
- if (!TLI->lowerInterleaveIntrinsicToStore(cast<Instruction>(StoredBy), Mask,
- InterleaveValues))
+ if (!TLI->lowerInterleaveIntrinsicToStore(StoredBy, Mask, InterleaveValues))
return false;
// We now have a target-specific store, so delete the old one.
- DeadInsts.insert(cast<Instruction>(StoredBy));
- DeadInsts.insert(II);
+ DeadInsts.insert(StoredBy);
+ DeadInsts.insert(IntII);
return true;
}
@@ -692,11 +687,13 @@ bool InterleavedAccessImpl::runOnFunction(Function &F) {
using namespace PatternMatch;
for (auto &I : instructions(F)) {
if (match(&I, m_CombineOr(m_Load(m_Value()),
- m_Intrinsic<Intrinsic::vp_load>())))
+ m_Intrinsic<Intrinsic::vp_load>())) ||
+ match(&I, m_Intrinsic<Intrinsic::masked_load>()))
Changed |= lowerInterleavedLoad(&I, DeadInsts);
if (match(&I, m_CombineOr(m_Store(m_Value(), m_Value()),
- m_Intrinsic<Intrinsic::vp_store>())))
+ m_Intrinsic<Intrinsic::vp_store>())) ||
+ match(&I, m_Intrinsic<Intrinsic::masked_store>()))
Changed |= lowerInterleavedStore(&I, DeadInsts);
if (auto *II = dyn_cast<IntrinsicInst>(&I)) {