diff options
author | Florian Hahn <flo@fhahn.com> | 2024-06-04 20:37:45 +0100 |
---|---|---|
committer | Florian Hahn <flo@fhahn.com> | 2024-06-04 20:37:46 +0100 |
commit | e775efcec476b3ec4eacfb14dc5bc7e86d9aa8fd (patch) | |
tree | f4a2059939615289f88adf9fd47a02b1f2f79a9f /llvm | |
parent | fbaec0f46b5853cbdbdfe77fb87e1321a1967956 (diff) | |
download | llvm-e775efcec476b3ec4eacfb14dc5bc7e86d9aa8fd.zip llvm-e775efcec476b3ec4eacfb14dc5bc7e86d9aa8fd.tar.gz llvm-e775efcec476b3ec4eacfb14dc5bc7e86d9aa8fd.tar.bz2 |
[LV] Apply loop guards when checking recur during hoisting RT checks.
Apply loop guards when checking if the recurrence is non-negative in
cases where runtime checks are hoisted out of an inner loop.
Diffstat (limited to 'llvm')
-rw-r--r-- | llvm/lib/Transforms/Utils/LoopUtils.cpp | 3 | ||||
-rw-r--r-- | llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll | 25 |
2 files changed, 13 insertions, 15 deletions
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index de3eb4a..922d4b6 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -1760,7 +1760,8 @@ static PointerBounds expandBounds(const RuntimeCheckingPtrGroup *CG, Low = cast<SCEVAddRecExpr>(Low)->getStart(); // If there is a possibility that the stride is negative then we have // to generate extra checks to ensure the stride is positive. - if (!SE.isKnownNonNegative(Recur)) { + if (!SE.isKnownNonNegative( + SE.applyLoopGuards(Recur, HighAR->getLoop()))) { Stride = Recur; LLVM_DEBUG(dbgs() << "LAA: ... but need to check stride is " "positive: " diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll index 8e6dbfc..e3e98c3 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll @@ -1509,7 +1509,8 @@ outer.exit: ret void } -; TODO: STRIDE_CHECK can be eliminated via loop guards. +; The stride for the access in the inner loop is known to be non-negative via +; loop guards. define void @stride_check_known_via_loop_guard(ptr %C, ptr %A, i32 %Acols) { ; CHECK-LABEL: define void @stride_check_known_via_loop_guard ; CHECK-SAME: (ptr [[C:%.*]], ptr [[A:%.*]], i32 [[ACOLS:%.*]]) { @@ -1518,8 +1519,6 @@ define void @stride_check_known_via_loop_guard(ptr %C, ptr %A, i32 %Acols) { ; CHECK-NEXT: br i1 [[PRE_C]], label [[EXIT:%.*]], label [[OUTER_HEADER_PREHEADER:%.*]] ; CHECK: outer.header.preheader: ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 8 -; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[ACOLS]] to i64 -; CHECK-NEXT: [[TMP1:%.*]] = shl nsw i64 [[TMP0]], 3 ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[C]], i64 34359738368 ; CHECK-NEXT: br label [[OUTER_HEADER:%.*]] ; CHECK: outer.header: @@ -1533,23 +1532,21 @@ define void @stride_check_known_via_loop_guard(ptr %C, ptr %A, i32 %Acols) { ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] -; CHECK-NEXT: [[STRIDE_CHECK:%.*]] = icmp slt i64 [[TMP1]], 0 -; CHECK-NEXT: [[TMP2:%.*]] = or i1 [[FOUND_CONFLICT]], [[STRIDE_CHECK]] -; CHECK-NEXT: br i1 [[TMP2]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] +; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 0 -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[C]], i32 [[TMP3]] -; CHECK-NEXT: [[TMP5:%.*]] = load double, ptr [[ARRAYIDX_US]], align 8, !alias.scope [[META69:![0-9]+]], !noalias [[META72:![0-9]+]] -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[C]], i32 [[TMP0]] +; CHECK-NEXT: [[TMP2:%.*]] = load double, ptr [[ARRAYIDX_US]], align 8, !alias.scope [[META69:![0-9]+]], !noalias [[META72:![0-9]+]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP2]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[TMP4]], i32 0 -; CHECK-NEXT: store <4 x double> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8, !alias.scope [[META72]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i32 0 +; CHECK-NEXT: store <4 x double> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 8, !alias.scope [[META72]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 0 -; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP74:![0-9]+]] +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP74:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: br i1 true, label [[OUTER_LATCH]], label [[SCALAR_PH]] ; CHECK: scalar.ph: |