From 00dc1b74b79a7294d4dc878b76960a0de8f40e38 Mon Sep 17 00:00:00 2001 From: Anna Thomas Date: Tue, 11 Apr 2017 21:02:00 +0000 Subject: [LV] Avoid vectorizing first order recurrence when phi uses are outside loop In the vectorization of first order recurrence, we vectorize such that the last element in the vector will be the one extracted to pass into the scalar remainder loop. However, this is not true when there is a phi (other than the primary induction variable) is used outside the loop. In such a case, we need the value from the second last iteration (i.e. the phi value), not the last iteration (which would be the phi update). I've added a test case for this. Also see PR32396. A follow up patch would generate the correct code gen for such cases, and turn this vectorization on. Differential Revision: https://reviews.llvm.org/D31910 Reviewers: mssimpso llvm-svn: 299985 --- llvm/lib/Transforms/Utils/LoopUtils.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'llvm/lib/Transforms/Utils/LoopUtils.cpp') diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index b5e6418..444bc16 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -553,13 +553,23 @@ bool RecurrenceDescriptor::isFirstOrderRecurrence(PHINode *Phi, Loop *TheLoop, if (!Previous || !TheLoop->contains(Previous) || isa(Previous)) return false; - // Ensure every user of the phi node is dominated by the previous value. The - // dominance requirement ensures the loop vectorizer will not need to - // vectorize the initial value prior to the first iteration of the loop. for (User *U : Phi->users()) - if (auto *I = dyn_cast(U)) + if (auto *I = dyn_cast(U)) { + // Ensure every user of the phi node is dominated by the previous value. + // The dominance requirement ensures the loop vectorizer will not need to + // vectorize the initial value prior to the first iteration of the loop. if (!DT->dominates(Previous, I)) return false; + // When the phi node has users outside the loop, the current logic for + // fixFirstOrderRecurrences may generate incorrect code. Specifically, we + // extract the last element from the vectorized phi, which would be the + // update to the phi before exiting the loop. However, what we want is the + // previous phi value before the update (i.e. the second last update + // before end of the vectorized loop). + // See added test cases in first-order-recurrence.ll + if (!TheLoop->contains(I)) + return false; + } return true; } -- cgit v1.1