From 42ebf3eaafc2a5c3c9338020186c0ad44cc4edf7 Mon Sep 17 00:00:00 2001 From: klensy Date: Wed, 5 Jun 2024 12:43:32 +0300 Subject: [PATCH 001/675] [utils][filecheck-lint]: speedup filecheck_lint (#94191) For example: clang\test\OpenMP\task_codegen.cpp: 0m29.570s -> 0m0.159s clang\test\Driver: 4m55.917s -> 1m48.053s Most win from big files. --------- Co-authored-by: klensy --- llvm/utils/filecheck_lint/filecheck_lint.py | 53 +++++++++++++------ .../filecheck_lint/filecheck_lint_test.py | 18 ++----- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/llvm/utils/filecheck_lint/filecheck_lint.py b/llvm/utils/filecheck_lint/filecheck_lint.py index 837846db8332..12f8299b8361 100755 --- a/llvm/utils/filecheck_lint/filecheck_lint.py +++ b/llvm/utils/filecheck_lint/filecheck_lint.py @@ -81,29 +81,40 @@ class FileRange: """Stores the coordinates of a span on a single line within a file. Attributes: - line: the line number - start_column: the (inclusive) column where the span starts - end_column: the (inclusive) column where the span ends + content: line str + start_byte: the (inclusive) byte offset the span starts + end_byte: the (inclusive) byte offset the span ends """ - line: int - start_column: int - end_column: int + content: str + start_byte: int + end_byte: int def __init__( self, content: str, start_byte: int, end_byte: int ): # pylint: disable=g-doc-args - """Derives a span's coordinates based on a string and start/end bytes. + """ + Stores the coordinates of a span based on a string and start/end bytes. `start_byte` and `end_byte` are assumed to be on the same line. """ - content_before_span = content[:start_byte] - self.line = content_before_span.count("\n") + 1 - self.start_column = start_byte - content_before_span.rfind("\n") - self.end_column = self.start_column + (end_byte - start_byte - 1) + self.content = content + self.start_byte = start_byte + self.end_byte = end_byte - def __str__(self) -> str: - return f"{self.line}:{self.start_column}-{self.end_column}" + def as_str(self): + """ + Derives span from line and coordinates. + + start_column: the (inclusive) column where the span starts + end_column: the (inclusive) column where the span ends + """ + content_before_span = self.content[: self.start_byte] + line = content_before_span.count("\n") + 1 + start_column = self.start_byte - content_before_span.rfind("\n") + end_column = start_column + (self.end_byte - self.start_byte - 1) + + return f"{line}:{start_column}-{end_column}" class Diagnostic: @@ -134,7 +145,7 @@ class Diagnostic: self.fix = fix def __str__(self) -> str: - return f"{self.filepath}:" + str(self.filerange) + f": {self.summary()}" + return f"{self.filepath}:" + self.filerange.as_str() + f": {self.summary()}" def summary(self) -> str: return ( @@ -228,7 +239,8 @@ def find_directive_typos( ) potential_directives = find_potential_directives(content) - + # Cache score and best_match to skip recalculating. + score_and_best_match_for_potential_directive = dict() for filerange, potential_directive in potential_directives: # TODO(bchetioui): match count directives more finely. We skip directives # starting with 'CHECK-COUNT-' for the moment as they require more complex @@ -244,7 +256,16 @@ def find_directive_typos( if len(potential_directive) > max(map(len, all_directives)) + threshold: continue - score, best_match = find_best_match(potential_directive) + if potential_directive not in score_and_best_match_for_potential_directive: + score, best_match = find_best_match(potential_directive) + score_and_best_match_for_potential_directive[potential_directive] = ( + score, + best_match, + ) + else: + score, best_match = score_and_best_match_for_potential_directive[ + potential_directive + ] if score == 0: # This is an actual directive, ignore. continue elif score <= threshold and best_match not in _ignore: diff --git a/llvm/utils/filecheck_lint/filecheck_lint_test.py b/llvm/utils/filecheck_lint/filecheck_lint_test.py index 16f381d5b045..6edcf0abd25a 100644 --- a/llvm/utils/filecheck_lint/filecheck_lint_test.py +++ b/llvm/utils/filecheck_lint/filecheck_lint_test.py @@ -49,27 +49,15 @@ class TestTypoDetection(unittest.TestCase): results = list(fcl.find_potential_directives(content)) assert len(results) == 3 pos, match = results[0] - assert ( - pos.line == 1 - and pos.start_column == len("junk; ") + 1 - and pos.end_column == len(lines[0]) - 1 - ) + assert pos.as_str() == "1:7-11" assert match == "CHCK1" pos, match = results[1] - assert ( - pos.line == 2 - and pos.start_column == len("junk// ") + 1 - and pos.end_column == len(lines[1]) - 1 - ) + assert pos.as_str() == "2:8-12" assert match == "CHCK2" pos, match = results[2] - assert ( - pos.line == 3 - and pos.start_column == 1 - and pos.end_column == len(lines[2]) - 1 - ) + assert pos.as_str() == "3:1-10" assert match == "SOME CHCK3" def test_levenshtein(self): -- GitLab From 043cc5a2275d014766dd4ec2ad4fe07d5516ceef Mon Sep 17 00:00:00 2001 From: Kerry McLaughlin Date: Wed, 5 Jun 2024 10:57:14 +0100 Subject: [PATCH 002/675] [AArch64][compiler-rt] Add a function returning the current vector length (#92921) __arm_get_current_vg emits a cntd instruction if in streaming mode or SVE is available at runtime, otherwise it will return 0. --- compiler-rt/lib/builtins/CMakeLists.txt | 2 +- compiler-rt/lib/builtins/aarch64/sme-abi-vg.c | 45 ++++++++++ compiler-rt/lib/builtins/cpu_model/aarch64.c | 70 +-------------- compiler-rt/lib/builtins/cpu_model/aarch64.h | 88 +++++++++++++++++++ 4 files changed, 135 insertions(+), 70 deletions(-) create mode 100644 compiler-rt/lib/builtins/aarch64/sme-abi-vg.c create mode 100644 compiler-rt/lib/builtins/cpu_model/aarch64.h diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt index c72eb337109c..0b9e9bd7a295 100644 --- a/compiler-rt/lib/builtins/CMakeLists.txt +++ b/compiler-rt/lib/builtins/CMakeLists.txt @@ -562,7 +562,7 @@ set(aarch64_SOURCES ) if(COMPILER_RT_HAS_AARCH64_SME AND COMPILER_RT_HAS_FNO_BUILTIN_FLAG AND (COMPILER_RT_HAS_AUXV OR COMPILER_RT_BAREMETAL_BUILD)) - list(APPEND aarch64_SOURCES aarch64/sme-abi.S aarch64/sme-abi-init.c aarch64/sme-libc-routines.c) + list(APPEND aarch64_SOURCES aarch64/sme-abi.S aarch64/sme-abi-init.c aarch64/sme-abi-vg.c aarch64/sme-libc-routines.c) message(STATUS "AArch64 SME ABI routines enabled") set_source_files_properties(aarch64/sme-libc-routines.c PROPERTIES COMPILE_FLAGS "-fno-builtin") else() diff --git a/compiler-rt/lib/builtins/aarch64/sme-abi-vg.c b/compiler-rt/lib/builtins/aarch64/sme-abi-vg.c new file mode 100644 index 000000000000..e384ab7f87c4 --- /dev/null +++ b/compiler-rt/lib/builtins/aarch64/sme-abi-vg.c @@ -0,0 +1,45 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "../cpu_model/aarch64.h" + +struct FEATURES { + long long features; +}; + +extern struct FEATURES __aarch64_cpu_features; + +struct SME_STATE { + long PSTATE; + long TPIDR2_EL0; +}; + +extern struct SME_STATE __arm_sme_state(void) __arm_streaming_compatible; + +extern bool __aarch64_has_sme_and_tpidr2_el0; + +#if __GNUC__ >= 9 +#pragma GCC diagnostic ignored "-Wprio-ctor-dtor" +#endif +__attribute__((constructor(90))) static void get_aarch64_cpu_features(void) { + if (!__aarch64_cpu_features.features) + __init_cpu_features(); +} + +__attribute__((target("sve"))) long +__arm_get_current_vg(void) __arm_streaming_compatible { + struct SME_STATE State = __arm_sme_state(); + bool HasSVE = __aarch64_cpu_features.features & (1ULL << FEAT_SVE); + + if (!HasSVE && !__aarch64_has_sme_and_tpidr2_el0) + return 0; + + if (HasSVE || (State.PSTATE & 1)) { + long vl; + __asm__ __volatile__("cntd %0" : "=r"(vl)); + return vl; + } + + return 0; +} diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64.c b/compiler-rt/lib/builtins/cpu_model/aarch64.c index 17bddfca46f0..b868caa991b2 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64.c +++ b/compiler-rt/lib/builtins/cpu_model/aarch64.c @@ -12,7 +12,7 @@ // //===----------------------------------------------------------------------===// -#include "cpu_model.h" +#include "aarch64.h" #if !defined(__aarch64__) #error This file is intended only for aarch64-based targets @@ -53,74 +53,6 @@ _Bool __aarch64_have_lse_atomics #endif #if !defined(DISABLE_AARCH64_FMV) -// CPUFeatures must correspond to the same AArch64 features in -// AArch64TargetParser.h -enum CPUFeatures { - FEAT_RNG, - FEAT_FLAGM, - FEAT_FLAGM2, - FEAT_FP16FML, - FEAT_DOTPROD, - FEAT_SM4, - FEAT_RDM, - FEAT_LSE, - FEAT_FP, - FEAT_SIMD, - FEAT_CRC, - FEAT_SHA1, - FEAT_SHA2, - FEAT_SHA3, - FEAT_AES, - FEAT_PMULL, - FEAT_FP16, - FEAT_DIT, - FEAT_DPB, - FEAT_DPB2, - FEAT_JSCVT, - FEAT_FCMA, - FEAT_RCPC, - FEAT_RCPC2, - FEAT_FRINTTS, - FEAT_DGH, - FEAT_I8MM, - FEAT_BF16, - FEAT_EBF16, - FEAT_RPRES, - FEAT_SVE, - FEAT_SVE_BF16, - FEAT_SVE_EBF16, - FEAT_SVE_I8MM, - FEAT_SVE_F32MM, - FEAT_SVE_F64MM, - FEAT_SVE2, - FEAT_SVE_AES, - FEAT_SVE_PMULL128, - FEAT_SVE_BITPERM, - FEAT_SVE_SHA3, - FEAT_SVE_SM4, - FEAT_SME, - FEAT_MEMTAG, - FEAT_MEMTAG2, - FEAT_MEMTAG3, - FEAT_SB, - FEAT_PREDRES, - FEAT_SSBS, - FEAT_SSBS2, - FEAT_BTI, - FEAT_LS64, - FEAT_LS64_V, - FEAT_LS64_ACCDATA, - FEAT_WFXT, - FEAT_SME_F64, - FEAT_SME_I64, - FEAT_SME2, - FEAT_RCPC3, - FEAT_MOPS, - FEAT_MAX, - FEAT_EXT = 62, // Reserved to indicate presence of additional features field - // in __aarch64_cpu_features - FEAT_INIT // Used as flag of features initialization completion -}; // Architecture features used // in Function Multi Versioning diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64.h b/compiler-rt/lib/builtins/cpu_model/aarch64.h new file mode 100644 index 000000000000..15d5300da53b --- /dev/null +++ b/compiler-rt/lib/builtins/cpu_model/aarch64.h @@ -0,0 +1,88 @@ +//===-- cpu_model/aarch64.h --------------------------------------------- -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "cpu_model.h" + +#if !defined(__aarch64__) +#error This file is intended only for aarch64-based targets +#endif + +#if !defined(DISABLE_AARCH64_FMV) + +// CPUFeatures must correspond to the same AArch64 features in +// AArch64TargetParser.h +enum CPUFeatures { + FEAT_RNG, + FEAT_FLAGM, + FEAT_FLAGM2, + FEAT_FP16FML, + FEAT_DOTPROD, + FEAT_SM4, + FEAT_RDM, + FEAT_LSE, + FEAT_FP, + FEAT_SIMD, + FEAT_CRC, + FEAT_SHA1, + FEAT_SHA2, + FEAT_SHA3, + FEAT_AES, + FEAT_PMULL, + FEAT_FP16, + FEAT_DIT, + FEAT_DPB, + FEAT_DPB2, + FEAT_JSCVT, + FEAT_FCMA, + FEAT_RCPC, + FEAT_RCPC2, + FEAT_FRINTTS, + FEAT_DGH, + FEAT_I8MM, + FEAT_BF16, + FEAT_EBF16, + FEAT_RPRES, + FEAT_SVE, + FEAT_SVE_BF16, + FEAT_SVE_EBF16, + FEAT_SVE_I8MM, + FEAT_SVE_F32MM, + FEAT_SVE_F64MM, + FEAT_SVE2, + FEAT_SVE_AES, + FEAT_SVE_PMULL128, + FEAT_SVE_BITPERM, + FEAT_SVE_SHA3, + FEAT_SVE_SM4, + FEAT_SME, + FEAT_MEMTAG, + FEAT_MEMTAG2, + FEAT_MEMTAG3, + FEAT_SB, + FEAT_PREDRES, + FEAT_SSBS, + FEAT_SSBS2, + FEAT_BTI, + FEAT_LS64, + FEAT_LS64_V, + FEAT_LS64_ACCDATA, + FEAT_WFXT, + FEAT_SME_F64, + FEAT_SME_I64, + FEAT_SME2, + FEAT_RCPC3, + FEAT_MOPS, + FEAT_MAX, + FEAT_EXT = 62, // Reserved to indicate presence of additional features field + // in __aarch64_cpu_features + FEAT_INIT // Used as flag of features initialization completion +}; + +void __init_cpu_features(void); + +#endif // !defined(DISABLE_AARCH64_FMV) -- GitLab From e635520be888335dd59874038d33e60cca3a7143 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 5 Jun 2024 10:58:54 +0100 Subject: [PATCH 003/675] [DAG] computeKnownBits - abs(x) will be zero in the upper bits if x is sign-extended (#94382) As reported on https://github.com/llvm/llvm-project/issues/94344 - if x has more than one signbit, then the upper bits of its absolute value are guaranteed to be zero Alive2: https://alive2.llvm.org/ce/z/a87fHU Fixes #94344 --- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 2 + llvm/test/CodeGen/X86/combine-abs.ll | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 414c724b94f7..6c9b64810c33 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -4051,6 +4051,8 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, case ISD::ABS: { Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known = Known2.abs(); + Known.Zero.setHighBits( + ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1) - 1); break; } case ISD::USUBSAT: { diff --git a/llvm/test/CodeGen/X86/combine-abs.ll b/llvm/test/CodeGen/X86/combine-abs.ll index 202c88109eae..76ee02e79870 100644 --- a/llvm/test/CodeGen/X86/combine-abs.ll +++ b/llvm/test/CodeGen/X86/combine-abs.ll @@ -201,6 +201,50 @@ define <8 x i32> @combine_v8i32_abs_pos(<8 x i32> %a) { ret <8 x i32> %2 } +; (abs x) upper bits are known zero if x has extra sign bits +define i32 @combine_i32_abs_zerosign(i32 %a) { +; CHECK-LABEL: combine_i32_abs_zerosign: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: retq + %1 = ashr i32 %a, 15 + %2 = call i32 @llvm.abs.i32(i32 %1, i1 false) + %3 = and i32 %2, -524288 ; 0xFFF80000 + ret i32 %3 +} + +define <8 x i16> @combine_v8i16_abs_zerosign(<8 x i16> %a) { +; SSE-LABEL: combine_v8i16_abs_zerosign: +; SSE: # %bb.0: +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: combine_v8i16_abs_zerosign: +; AVX: # %bb.0: +; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = ashr <8 x i16> %a, + %2 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %1, i1 false) + %3 = and <8 x i16> %2, + ret <8 x i16> %3 +} + +; negative test - mask extends beyond known zero bits +define i32 @combine_i32_abs_zerosign_negative(i32 %a) { +; CHECK-LABEL: combine_i32_abs_zerosign_negative: +; CHECK: # %bb.0: +; CHECK-NEXT: sarl $3, %edi +; CHECK-NEXT: movl %edi, %eax +; CHECK-NEXT: negl %eax +; CHECK-NEXT: cmovsl %edi, %eax +; CHECK-NEXT: andl $536346624, %eax # imm = 0x1FF80000 +; CHECK-NEXT: retq + %1 = ashr i32 %a, 3 + %2 = call i32 @llvm.abs.i32(i32 %1, i1 false) + %3 = and i32 %2, -524288 ; 0xFFF80000 + ret i32 %3 +} + declare <16 x i8> @llvm.abs.v16i8(<16 x i8>, i1) nounwind readnone declare <4 x i32> @llvm.abs.v4i32(<4 x i32>, i1) nounwind readnone declare <8 x i16> @llvm.abs.v8i16(<8 x i16>, i1) nounwind readnone -- GitLab From 05e1b5340b0caf19ef2f8323b84082c389850720 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 5 Jun 2024 11:18:06 +0100 Subject: [PATCH 004/675] [VPlan] Model FOR resume value extraction in VPlan. (#93396) This patch uses the ExtractFromEnd VPInstruction opcode to extract the value of a FOR to be used as resume value for the ph in the scalar loop. It adds a new live-out that temporarily wraps the FOR phi in the scalar loop. fixFixedOrderRecurrence will process live outs for fixed order recurrence phis by creating a new phi node in the scalar preheader, using the generated value for the live-out as incoming value from the middle block and the original start value as incoming value for the other edge. Creation of the phi in the preheader, as well as updating the phi in the scalar loop will also be moved to VPlan in the future, eventually retiring fixFixedOrderRecurrence Depends on https://github.com/llvm/llvm-project/pull/93395 PR: https://github.com/llvm/llvm-project/pull/93396 --- .../Transforms/Vectorize/LoopVectorize.cpp | 119 +++++------------- llvm/lib/Transforms/Vectorize/VPlan.h | 4 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 81 +++++++++++- .../AArch64/fixed-order-recurrence.ll | 4 +- .../LoopVectorize/AArch64/induction-costs.ll | 2 +- .../AArch64/loop-vectorization-factors.ll | 4 +- .../AArch64/reduction-recurrence-costs-sve.ll | 2 +- .../AArch64/sve-interleaved-accesses.ll | 2 +- .../X86/fixed-order-recurrence.ll | 4 +- .../Transforms/LoopVectorize/X86/pr72969.ll | 2 +- .../first-order-recurrence-chains-vplan.ll | 12 ++ .../first-order-recurrence-chains.ll | 30 ++--- .../first-order-recurrence-complex.ll | 4 +- ...-order-recurrence-sink-replicate-region.ll | 17 +++ .../LoopVectorize/first-order-recurrence.ll | 54 ++++---- .../Transforms/LoopVectorize/induction.ll | 10 +- .../interleave-and-scalarize-only.ll | 3 + .../LoopVectorize/interleaved-accesses.ll | 2 +- .../LoopVectorize/vplan-printing.ll | 2 + 19 files changed, 206 insertions(+), 152 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 188bfc164f30..c7c19ef456c7 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -59,6 +59,7 @@ #include "VPlan.h" #include "VPlanAnalysis.h" #include "VPlanHCFGBuilder.h" +#include "VPlanPatternMatch.h" #include "VPlanTransforms.h" #include "VPlanVerifier.h" #include "llvm/ADT/APInt.h" @@ -606,10 +607,9 @@ protected: BasicBlock *MiddleBlock, BasicBlock *VectorHeader, VPlan &Plan, VPTransformState &State); - /// Create the exit value of first order recurrences in the middle block and - /// update their users. - void fixFixedOrderRecurrence(VPFirstOrderRecurrencePHIRecipe *PhiR, - VPTransformState &State); + /// Create the phi node for the resume value of first order recurrences in the + /// scalar preheader and update the users in the scalar loop. + void fixFixedOrderRecurrence(VPLiveOut *LO, VPTransformState &State); /// Iteratively sink the scalarized operands of a predicated instruction into /// the block that was created for it. @@ -3391,16 +3391,16 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State, fixNonInductionPHIs(Plan, State); // At this point every instruction in the original loop is widened to a - // vector form. Now we need to fix the recurrences in the loop. These PHI - // nodes are currently empty because we did not want to introduce cycles. - // This is the second stage of vectorizing recurrences. Note that fixing - // reduction phis are already modeled in VPlan. - // TODO: Also model fixing fixed-order recurrence phis in VPlan. - VPRegionBlock *VectorRegion = State.Plan->getVectorLoopRegion(); - VPBasicBlock *HeaderVPBB = VectorRegion->getEntryBasicBlock(); - for (VPRecipeBase &R : HeaderVPBB->phis()) { - if (auto *FOR = dyn_cast(&R)) - fixFixedOrderRecurrence(FOR, State); + // vector form. Note that fixing reduction phis, as well as extracting the + // exit and resume values for fixed-order recurrences are already modeled in + // VPlan. All that remains to do here is to create a phi in the scalar + // pre-header for each fixed-order recurrence resume value. + // TODO: Also model creating phis in the scalar pre-header in VPlan. + for (const auto &[_, LO] : to_vector(Plan.getLiveOuts())) { + if (!Legal->isFixedOrderRecurrence(LO->getPhi())) + continue; + fixFixedOrderRecurrence(LO, State); + Plan.removeLiveOut(LO->getPhi()); } // Forget the original basic block. @@ -3416,6 +3416,7 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State, for (PHINode &PN : Exit->phis()) PSE.getSE()->forgetLcssaPhiWithNewPredecessor(OrigLoop, &PN); + VPRegionBlock *VectorRegion = State.Plan->getVectorLoopRegion(); VPBasicBlock *LatchVPBB = VectorRegion->getExitingBasicBlock(); Loop *VectorLoop = LI->getLoopFor(State.CFG.VPBB2IRBB[LatchVPBB]); if (Cost->requiresScalarEpilogue(VF.isVector())) { @@ -3469,85 +3470,31 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State, VF.getKnownMinValue() * UF); } -void InnerLoopVectorizer::fixFixedOrderRecurrence( - VPFirstOrderRecurrencePHIRecipe *PhiR, VPTransformState &State) { - // This is the second phase of vectorizing first-order recurrences. An - // overview of the transformation is described below. Suppose we have the - // following loop. - // - // for (int i = 0; i < n; ++i) - // b[i] = a[i] - a[i - 1]; - // - // There is a first-order recurrence on "a". For this loop, the shorthand - // scalar IR looks like: - // - // scalar.ph: - // s_init = a[-1] - // br scalar.body - // - // scalar.body: - // i = phi [0, scalar.ph], [i+1, scalar.body] - // s1 = phi [s_init, scalar.ph], [s2, scalar.body] - // s2 = a[i] - // b[i] = s2 - s1 - // br cond, scalar.body, ... - // - // In this example, s1 is a recurrence because it's value depends on the - // previous iteration. In the first phase of vectorization, we created a - // vector phi v1 for s1. We now complete the vectorization and produce the - // shorthand vector IR shown below (for VF = 4, UF = 1). - // - // vector.ph: - // v_init = vector(..., ..., ..., a[-1]) - // br vector.body - // - // vector.body - // i = phi [0, vector.ph], [i+4, vector.body] - // v1 = phi [v_init, vector.ph], [v2, vector.body] - // v2 = a[i, i+1, i+2, i+3]; - // v3 = vector(v1(3), v2(0, 1, 2)) - // b[i, i+1, i+2, i+3] = v2 - v3 - // br cond, vector.body, middle.block - // - // middle.block: - // x = v2(3) - // br scalar.ph - // - // scalar.ph: - // s_init = phi [x, middle.block], [a[-1], otherwise] - // br scalar.body - // - // After execution completes the vector loop, we extract the next value of - // the recurrence (x) to use as the initial value in the scalar loop. - +void InnerLoopVectorizer::fixFixedOrderRecurrence(VPLiveOut *LO, + VPTransformState &State) { // Extract the last vector element in the middle block. This will be the // initial value for the recurrence when jumping to the scalar loop. - VPValue *PreviousDef = PhiR->getBackedgeValue(); - Value *Incoming = State.get(PreviousDef, UF - 1); - auto *ExtractForScalar = Incoming; - auto *IdxTy = Builder.getInt32Ty(); - Value *RuntimeVF = nullptr; - if (VF.isVector()) { - auto *One = ConstantInt::get(IdxTy, 1); - Builder.SetInsertPoint(LoopMiddleBlock->getTerminator()); - RuntimeVF = getRuntimeVF(Builder, IdxTy, VF); - auto *LastIdx = Builder.CreateSub(RuntimeVF, One); - ExtractForScalar = - Builder.CreateExtractElement(Incoming, LastIdx, "vector.recur.extract"); - } + VPValue *VPExtract = LO->getOperand(0); + using namespace llvm::VPlanPatternMatch; + assert(match(VPExtract, m_VPInstruction( + m_VPValue(), m_VPValue())) && + "FOR LiveOut expects to use an extract from end."); + Value *ResumeScalarFOR = State.get(VPExtract, UF - 1, true); // Fix the initial value of the original recurrence in the scalar loop. + PHINode *ScalarHeaderPhi = LO->getPhi(); + auto *InitScalarFOR = + ScalarHeaderPhi->getIncomingValueForBlock(LoopScalarPreHeader); Builder.SetInsertPoint(LoopScalarPreHeader, LoopScalarPreHeader->begin()); - PHINode *Phi = cast(PhiR->getUnderlyingValue()); - auto *Start = Builder.CreatePHI(Phi->getType(), 2, "scalar.recur.init"); - auto *ScalarInit = PhiR->getStartValue()->getLiveInIRValue(); + auto *ScalarPreheaderPhi = + Builder.CreatePHI(ScalarHeaderPhi->getType(), 2, "scalar.recur.init"); for (auto *BB : predecessors(LoopScalarPreHeader)) { - auto *Incoming = BB == LoopMiddleBlock ? ExtractForScalar : ScalarInit; - Start->addIncoming(Incoming, BB); + auto *Incoming = BB == LoopMiddleBlock ? ResumeScalarFOR : InitScalarFOR; + ScalarPreheaderPhi->addIncoming(Incoming, BB); } - - Phi->setIncomingValueForBlock(LoopScalarPreHeader, Start); - Phi->setName("scalar.recur"); + ScalarHeaderPhi->setIncomingValueForBlock(LoopScalarPreHeader, + ScalarPreheaderPhi); + ScalarHeaderPhi->setName("scalar.recur"); } void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index bd500728883b..943edc352086 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3166,7 +3166,9 @@ class VPlan { /// definitions are VPValues that hold a pointer to their underlying IR. SmallVector VPLiveInsToFree; - /// Values used outside the plan. + /// Values used outside the plan. It contains live-outs that need fixing. Any + /// live-out that is fixed outside VPlan needs to be removed. The remaining + /// live-outs are fixed via VPLiveOut::fixPhi. MapVector LiveOuts; /// Mapping from SCEVs to the VPValues representing their expansions. diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index af3cf0ad7af0..ab3b5cf2b9da 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -847,14 +847,91 @@ bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan, // all users. RecurSplice->setOperand(0, FOR); + // This is the second phase of vectorizing first-order recurrences. An + // overview of the transformation is described below. Suppose we have the + // following loop with some use after the loop of the last a[i-1], + // + // for (int i = 0; i < n; ++i) { + // t = a[i - 1]; + // b[i] = a[i] - t; + // } + // use t; + // + // There is a first-order recurrence on "a". For this loop, the shorthand + // scalar IR looks like: + // + // scalar.ph: + // s_init = a[-1] + // br scalar.body + // + // scalar.body: + // i = phi [0, scalar.ph], [i+1, scalar.body] + // s1 = phi [s_init, scalar.ph], [s2, scalar.body] + // s2 = a[i] + // b[i] = s2 - s1 + // br cond, scalar.body, exit.block + // + // exit.block: + // use = lcssa.phi [s1, scalar.body] + // + // In this example, s1 is a recurrence because it's value depends on the + // previous iteration. In the first phase of vectorization, we created a + // vector phi v1 for s1. We now complete the vectorization and produce the + // shorthand vector IR shown below (for VF = 4, UF = 1). + // + // vector.ph: + // v_init = vector(..., ..., ..., a[-1]) + // br vector.body + // + // vector.body + // i = phi [0, vector.ph], [i+4, vector.body] + // v1 = phi [v_init, vector.ph], [v2, vector.body] + // v2 = a[i, i+1, i+2, i+3]; + // v3 = vector(v1(3), v2(0, 1, 2)) + // b[i, i+1, i+2, i+3] = v2 - v3 + // br cond, vector.body, middle.block + // + // middle.block: + // s_penultimate = v2(2) = v3(3) + // s_resume = v2(3) + // br cond, scalar.ph, exit.block + // + // scalar.ph: + // s_init' = phi [s_resume, middle.block], [s_init, otherwise] + // br scalar.body + // + // scalar.body: + // i = phi [0, scalar.ph], [i+1, scalar.body] + // s1 = phi [s_init', scalar.ph], [s2, scalar.body] + // s2 = a[i] + // b[i] = s2 - s1 + // br cond, scalar.body, exit.block + // + // exit.block: + // lo = lcssa.phi [s1, scalar.body], [s.penultimate, middle.block] + // + // After execution completes the vector loop, we extract the next value of + // the recurrence (x) to use as the initial value in the scalar loop. This + // is modeled by ExtractFromEnd. Type *IntTy = Plan.getCanonicalIV()->getScalarType(); - auto *Result = cast(MiddleBuilder.createNaryOp( + + // Extract the penultimate value of the recurrence and update VPLiveOut + // users of the recurrence splice. + auto *Penultimate = cast(MiddleBuilder.createNaryOp( VPInstruction::ExtractFromEnd, {FOR->getBackedgeValue(), Plan.getOrAddLiveIn(ConstantInt::get(IntTy, 2))}, {}, "vector.recur.extract.for.phi")); RecurSplice->replaceUsesWithIf( - Result, [](VPUser &U, unsigned) { return isa(&U); }); + Penultimate, [](VPUser &U, unsigned) { return isa(&U); }); + + // Extract the resume value and create a new VPLiveOut for it. + auto *Resume = MiddleBuilder.createNaryOp( + VPInstruction::ExtractFromEnd, + {FOR->getBackedgeValue(), + Plan.getOrAddLiveIn(ConstantInt::get(IntTy, 1))}, + {}, "vector.recur.extract"); + Plan.addLiveOut(cast(FOR->getUnderlyingInstr()), Resume); } return true; } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-order-recurrence.ll index 33d7a3a3c8ac..50ab61cac5b1 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-order-recurrence.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-order-recurrence.ll @@ -47,8 +47,8 @@ define void @firstorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i8> [[WIDE_LOAD1]], i32 15 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -154,10 +154,10 @@ define void @thirdorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt ; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i8> [[WIDE_LOAD5]], i32 15 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT6:%.*]] = extractelement <16 x i8> [[TMP8]], i32 15 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT9:%.*]] = extractelement <16 x i8> [[TMP10]], i32 15 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT10:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT9]], [[MIDDLE_BLOCK]] ] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll index 162fb9c802dd..32dc2ec1d50a 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll @@ -127,8 +127,8 @@ define i64 @pointer_induction_only(ptr %start, ptr %end) { ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0 -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll index 718148a67fcc..b7463032cada 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll @@ -786,8 +786,8 @@ define void @add_phifail(ptr noalias nocapture readonly %p, ptr noalias nocaptur ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP4]], i32 15 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -871,8 +871,8 @@ define i8 @add_phifail2(ptr noalias nocapture readonly %p, ptr noalias nocapture ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <16 x i32> [[TMP6]], i32 14 -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP6]], i32 15 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll index c24c1a38177d..1353290fa2e2 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll @@ -100,7 +100,6 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2) ; DEFAULT: middle.block: ; DEFAULT-NEXT: [[BIN_RDX:%.*]] = or [[TMP58]], [[TMP57]] ; DEFAULT-NEXT: [[TMP60:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32( [[BIN_RDX]]) -; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; DEFAULT-NEXT: [[TMP61:%.*]] = call i32 @llvm.vscale.i32() ; DEFAULT-NEXT: [[TMP62:%.*]] = mul i32 [[TMP61]], 4 ; DEFAULT-NEXT: [[TMP63:%.*]] = sub i32 [[TMP62]], 1 @@ -109,6 +108,7 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2) ; DEFAULT-NEXT: [[TMP65:%.*]] = mul i32 [[TMP64]], 4 ; DEFAULT-NEXT: [[TMP66:%.*]] = sub i32 [[TMP65]], 1 ; DEFAULT-NEXT: [[VECTOR_RECUR_EXTRACT13:%.*]] = extractelement [[TMP20]], i32 [[TMP66]] +; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; DEFAULT-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; DEFAULT: scalar.ph: ; DEFAULT-NEXT: [[SCALAR_RECUR_INIT14:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT13]], [[MIDDLE_BLOCK]] ] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll index 1853e551806b..3a25ffe26cc0 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll @@ -1509,11 +1509,11 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) #1 { ; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]] ; CHECK-NEXT: [[TMP30:%.*]] = call i32 @llvm.vscale.i32() ; CHECK-NEXT: [[TMP31:%.*]] = shl nuw nsw i32 [[TMP30]], 2 ; CHECK-NEXT: [[TMP32:%.*]] = add nsw i32 [[TMP31]], -1 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement [[WIDE_MASKED_GATHER4]], i32 [[TMP32]] +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] diff --git a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll index 8004563f3816..94575004e364 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll @@ -47,8 +47,8 @@ define void @firstorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i8> [[WIDE_LOAD1]], i32 15 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -154,10 +154,10 @@ define void @thirdorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt ; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i8> [[WIDE_LOAD5]], i32 15 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT6:%.*]] = extractelement <16 x i8> [[TMP8]], i32 15 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT9:%.*]] = extractelement <16 x i8> [[TMP10]], i32 15 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT10:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT9]], [[MIDDLE_BLOCK]] ] diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll b/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll index 45af544f0304..829a91f18e5e 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll @@ -83,8 +83,8 @@ define void @test(ptr %p) { ; VEC-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VEC-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; VEC: middle.block: -; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]] ; VEC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP28]], i32 3 +; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]] ; VEC-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; VEC: scalar.ph: ; VEC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll index c04178a1c13e..5907c58365fa 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll @@ -33,7 +33,12 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%for.1.next>, ir<1> +; CHECK-NEXT: EMIT vp<[[RESUME_2:%.+]]> = extract-from-end vp<[[FOR1_SPLICE]]>, ir<1> ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: Live-out i16 %for.1 = vp<[[RESUME_1]]> +; CHECK-NEXT: Live-out i16 %for.2 = vp<[[RESUME_2]]> ; CHECK-NEXT: } ; entry: @@ -89,7 +94,14 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%for.1.next>, ir<1> +; CHECK-NEXT: EMIT vp<[[RESUME_2:%.+]]> = extract-from-end vp<[[FOR1_SPLICE]]>, ir<1> +; CHECK-NEXT: EMIT vp<[[RESUME_3:%.+]]> = extract-from-end vp<[[FOR2_SPLICE]]>, ir<1> ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: Live-out i16 %for.1 = vp<[[RESUME_1]]> +; CHECK-NEXT: Live-out i16 %for.2 = vp<[[RESUME_2]]> +; CHECK-NEXT: Live-out i16 %for.3 = vp<[[RESUME_3]]> ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll index a1173c6b46a2..447f0b0bfee2 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll @@ -19,8 +19,8 @@ define i16 @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-NEXT: br i1 [[TMP8]], label %middle.block, label %vector.body ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3 ; entry: @@ -62,8 +62,8 @@ define i16 @test_chained_first_order_recurrences_2(ptr %ptr) { ; CHECK-NEXT: br i1 [[TMP8]], label %middle.block, label %vector.body, !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 ; entry: @@ -108,10 +108,10 @@ define i16 @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-NEXT: br i1 [[TMP10]], label %middle.block, label %vector.body, !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i16> [[TMP5]], i32 3 ; entry: @@ -220,10 +220,10 @@ define i16 @test_chained_first_order_recurrences_3_reordered_1(ptr %ptr) { ; CHECK-NEXT: br i1 [[TMP10]], label %middle.block, label %vector.body, !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i16> [[TMP5]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 ; entry: @@ -271,10 +271,10 @@ define i16 @test_chained_first_order_recurrences_3_reordered_2(ptr %ptr) { ; CHECK-NEXT: br i1 [[TMP10]], label %middle.block, label %vector.body, !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i16> [[TMP5]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 ; entry: @@ -322,10 +322,10 @@ define i16 @test_chained_first_order_recurrences_3_for2_no_other_uses(ptr %ptr) ; CHECK-NEXT: br i1 [[TMP10]], label %middle.block, label %vector.body, !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i16> [[TMP5]], i32 3 ; entry: @@ -372,10 +372,10 @@ define i16 @test_chained_first_order_recurrences_3_for1_for2_no_other_uses(ptr % ; CHECK-NEXT: br i1 [[TMP10]], label %middle.block, label %vector.body, !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i16> [[TMP5]], i32 3 ; entry: @@ -421,8 +421,8 @@ define double @test_chained_first_order_recurrence_sink_users_1(ptr %ptr) { ; CHECK-NEXT: br i1 [[TMP9]], label %middle.block, label %vector.body, !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x double> [[WIDE_LOAD]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x double> [[TMP4]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[WIDE_LOAD]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x double> [[TMP4]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x double> [[TMP4]], i32 3 ; entry: @@ -661,10 +661,10 @@ define double @test_resinking_required(ptr %p, ptr noalias %a, ptr noalias %b) { ; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x double> [[BROADCAST_SPLAT]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI6:%.*]] = extractelement <4 x double> [[BROADCAST_SPLAT4]], i32 2 -; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI10:%.*]] = extractelement <4 x double> [[TMP4]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[BROADCAST_SPLAT]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI6:%.*]] = extractelement <4 x double> [[BROADCAST_SPLAT4]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT5:%.*]] = extractelement <4 x double> [[BROADCAST_SPLAT4]], i32 3 +; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI10:%.*]] = extractelement <4 x double> [[TMP4]], i32 2 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT9:%.*]] = extractelement <4 x double> [[TMP4]], i32 3 ; CHECK-NEXT: br i1 true, label %End, label %scalar.ph ; diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll index 149e4705885b..6b1d38142b9e 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll @@ -854,8 +854,8 @@ define void @sink_dominance(ptr %ptr, i32 %N) { ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[UMAX1]], [[N_VEC]] ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP5]], i32 3 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[UMAX1]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -937,8 +937,8 @@ define void @sink_dominance_2(ptr %ptr, i32 %N) { ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[UMAX1]], [[N_VEC]] ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP5]], i32 3 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[UMAX1]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll index 116d0f65c235..30a98db665cb 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -73,7 +73,10 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%conv>, ir<1> ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: Live-out i32 %0 = vp<[[RESUME_1]]> ; CHECK-NEXT: } ; entry: @@ -145,7 +148,10 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%recur.next>, ir<1> ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: Live-out i32 %recur = vp<[[RESUME_1]]> ; CHECK-NEXT: } ; entry: @@ -216,9 +222,11 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: ; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>, vp<[[SEL]]> +; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%recur.next>, ir<1> ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out i32 %res = vp<[[RED_RES]]> +; CHECK-NEXT: Live-out i32 %recur = vp<[[RESUME_1]]> ; CHECK-NEXT: } ; entry: @@ -314,7 +322,10 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%conv>, ir<1> ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: Live-out i32 %0 = vp<[[RESUME_1]]> ; CHECK-NEXT: } ; entry: @@ -397,7 +408,10 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%recur.next>, ir<1> ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: Live-out i32 %recur = vp<[[RESUME_1]]> ; CHECK-NEXT: } ; entry: @@ -469,7 +483,10 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%l>, ir<1> ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: Live-out i32 %.pn = vp<[[RESUME_1]]> ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll index a588d71a2c76..6d3654d3b97e 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll @@ -55,8 +55,8 @@ define void @recurrence_1(ptr readonly noalias %a, ptr noalias %b, i32 %n) { ; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; UNROLL-NO-IC: middle.block: -; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD1]], i32 3 +; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] ; UNROLL-NO-IC: scalar.ph: ; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -165,8 +165,8 @@ define void @recurrence_1(ptr readonly noalias %a, ptr noalias %b, i32 %n) { ; SINK-AFTER-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SINK-AFTER-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; SINK-AFTER: middle.block: -; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3 +; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]] ; SINK-AFTER: scalar.ph: ; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -267,8 +267,8 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) { ; UNROLL-NO-IC: middle.block: ; UNROLL-NO-IC-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[TMP17]], <4 x i32> [[TMP18]]) ; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[RDX_MINMAX]]) -; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i32 3 +; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; UNROLL-NO-IC: scalar.ph: ; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -400,8 +400,8 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) { ; SINK-AFTER-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; SINK-AFTER: middle.block: ; SINK-AFTER-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP9]]) -; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3 +; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; SINK-AFTER: scalar.ph: ; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -530,8 +530,8 @@ define void @recurrence_3(ptr readonly noalias %a, ptr noalias %b, i32 %n, float ; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; UNROLL-NO-IC: middle.block: -; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i32 3 +; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; UNROLL-NO-IC: scalar.ph: ; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -674,8 +674,8 @@ define void @recurrence_3(ptr readonly noalias %a, ptr noalias %b, i32 %n, float ; SINK-AFTER-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SINK-AFTER-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; SINK-AFTER: middle.block: -; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 +; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; SINK-AFTER: scalar.ph: ; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP0]], [[FOR_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -919,8 +919,8 @@ define i32 @PR27246() { ; UNROLL-NO-IC-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; UNROLL-NO-IC: middle.block: ; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[STEP_ADD]], i32 2 -; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[I_016]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STEP_ADD]], i32 3 +; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[I_016]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP3]], label [[SCALAR_PH]] ; UNROLL-NO-IC: scalar.ph: ; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[E_015]], [[FOR_COND1_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -1013,8 +1013,8 @@ define i32 @PR27246() { ; SINK-AFTER-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; SINK-AFTER: middle.block: ; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[VEC_IND]], i32 2 -; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[I_016]], [[N_VEC]] ; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[VEC_IND]], i32 3 +; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[I_016]], [[N_VEC]] ; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP3]], label [[SCALAR_PH]] ; SINK-AFTER: scalar.ph: ; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[E_015]], [[FOR_COND1_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -1115,11 +1115,11 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) { ; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = load i32, ptr [[TMP23]], align 4 ; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = load i32, ptr [[TMP24]], align 4 ; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = load i32, ptr [[TMP25]], align 4 -; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = load i32, ptr [[TMP26]], align 4 +; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = load i32, ptr [[TMP26]], align 4 ; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = insertelement <4 x i32> poison, i32 [[TMP35]], i32 0 ; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP39]], i32 [[TMP36]], i32 1 ; UNROLL-NO-IC-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[VECTOR_RECUR_EXTRACT_FOR_PHI]], i32 2 -; UNROLL-NO-IC-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> [[TMP41]], i32 [[TMP38]], i32 3 +; UNROLL-NO-IC-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> [[TMP41]], i32 [[VECTOR_RECUR_EXTRACT]], i32 3 ; UNROLL-NO-IC-NEXT: [[TMP43:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP34]], <4 x i32> ; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = shufflevector <4 x i32> [[TMP34]], <4 x i32> [[TMP42]], <4 x i32> ; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 @@ -1127,7 +1127,6 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) { ; UNROLL-NO-IC-NEXT: br i1 [[TMP45]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; UNROLL-NO-IC: middle.block: ; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] -; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP42]], i32 3 ; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; UNROLL-NO-IC: scalar.ph: ; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -1223,18 +1222,17 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) { ; SINK-AFTER-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP11]], align 4 ; SINK-AFTER-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP12]], align 4 ; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = load i32, ptr [[TMP13]], align 4 -; SINK-AFTER-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP14]], align 4 +; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = load i32, ptr [[TMP14]], align 4 ; SINK-AFTER-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> poison, i32 [[TMP15]], i32 0 ; SINK-AFTER-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP16]], i32 1 ; SINK-AFTER-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[VECTOR_RECUR_EXTRACT_FOR_PHI]], i32 2 -; SINK-AFTER-NEXT: [[TMP22]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP18]], i32 3 +; SINK-AFTER-NEXT: [[TMP22]] = insertelement <4 x i32> [[TMP21]], i32 [[VECTOR_RECUR_EXTRACT]], i32 3 ; SINK-AFTER-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> ; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; SINK-AFTER-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SINK-AFTER-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; SINK-AFTER: middle.block: ; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] -; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP22]], i32 3 ; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; SINK-AFTER: scalar.ph: ; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[PRE_LOAD]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -1574,11 +1572,11 @@ define i32 @PR33613(ptr %b, double %j, i32 %d) { ; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = load double, ptr [[TMP12]], align 8 ; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = load double, ptr [[TMP13]], align 8 ; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = load double, ptr [[TMP14]], align 8 -; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = load double, ptr [[TMP15]], align 8 +; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = load double, ptr [[TMP15]], align 8 ; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = insertelement <4 x double> poison, double [[TMP24]], i32 0 ; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = insertelement <4 x double> [[TMP28]], double [[TMP25]], i32 1 ; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = insertelement <4 x double> [[TMP29]], double [[TMP26]], i32 2 -; UNROLL-NO-IC-NEXT: [[TMP31]] = insertelement <4 x double> [[TMP30]], double [[TMP27]], i32 3 +; UNROLL-NO-IC-NEXT: [[TMP31]] = insertelement <4 x double> [[TMP30]], double [[VECTOR_RECUR_EXTRACT]], i32 3 ; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP23]], <4 x i32> ; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = shufflevector <4 x double> [[TMP23]], <4 x double> [[TMP31]], <4 x i32> ; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = fmul <4 x double> [[TMP32]], [[TMP23]] @@ -1595,7 +1593,6 @@ define i32 @PR33613(ptr %b, double %j, i32 %d) { ; UNROLL-NO-IC: middle.block: ; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP41]], [[TMP40]] ; UNROLL-NO-IC-NEXT: [[TMP43:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) -; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP31]], i32 3 ; UNROLL-NO-IC-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] ; UNROLL-NO-IC: scalar.ph: ; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -1710,11 +1707,11 @@ define i32 @PR33613(ptr %b, double %j, i32 %d) { ; SINK-AFTER-NEXT: [[TMP8:%.*]] = load double, ptr [[TMP4]], align 8 ; SINK-AFTER-NEXT: [[TMP9:%.*]] = load double, ptr [[TMP5]], align 8 ; SINK-AFTER-NEXT: [[TMP10:%.*]] = load double, ptr [[TMP6]], align 8 -; SINK-AFTER-NEXT: [[TMP11:%.*]] = load double, ptr [[TMP7]], align 8 +; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = load double, ptr [[TMP7]], align 8 ; SINK-AFTER-NEXT: [[TMP12:%.*]] = insertelement <4 x double> poison, double [[TMP8]], i32 0 ; SINK-AFTER-NEXT: [[TMP13:%.*]] = insertelement <4 x double> [[TMP12]], double [[TMP9]], i32 1 ; SINK-AFTER-NEXT: [[TMP14:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP10]], i32 2 -; SINK-AFTER-NEXT: [[TMP15]] = insertelement <4 x double> [[TMP14]], double [[TMP11]], i32 3 +; SINK-AFTER-NEXT: [[TMP15]] = insertelement <4 x double> [[TMP14]], double [[VECTOR_RECUR_EXTRACT]], i32 3 ; SINK-AFTER-NEXT: [[TMP16:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP15]], <4 x i32> ; SINK-AFTER-NEXT: [[TMP17:%.*]] = fmul <4 x double> [[TMP16]], [[TMP15]] ; SINK-AFTER-NEXT: [[TMP18:%.*]] = fcmp une <4 x double> [[TMP17]], zeroinitializer @@ -1725,7 +1722,6 @@ define i32 @PR33613(ptr %b, double %j, i32 %d) { ; SINK-AFTER-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] ; SINK-AFTER: middle.block: ; SINK-AFTER-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP20]]) -; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP15]], i32 3 ; SINK-AFTER-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] ; SINK-AFTER: scalar.ph: ; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -1826,8 +1822,8 @@ define void @sink_after(ptr noalias %a, ptr noalias %b, i64 %n) { ; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] ; UNROLL-NO-IC: middle.block: -; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i32 3 +; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; UNROLL-NO-IC: scalar.ph: ; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -1934,8 +1930,8 @@ define void @sink_after(ptr noalias %a, ptr noalias %b, i64 %n) { ; SINK-AFTER-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SINK-AFTER-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] ; SINK-AFTER: middle.block: -; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 +; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; SINK-AFTER: scalar.ph: ; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -2042,11 +2038,11 @@ define void @PR34711(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %n) { ; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = load i16, ptr [[TMP14]], align 2 ; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = load i16, ptr [[TMP15]], align 2 ; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = load i16, ptr [[TMP16]], align 2 -; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = load i16, ptr [[TMP17]], align 2 +; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = load i16, ptr [[TMP17]], align 2 ; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = insertelement <4 x i16> poison, i16 [[TMP28]], i32 0 ; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = insertelement <4 x i16> [[TMP32]], i16 [[TMP29]], i32 1 ; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = insertelement <4 x i16> [[TMP33]], i16 [[TMP30]], i32 2 -; UNROLL-NO-IC-NEXT: [[TMP35]] = insertelement <4 x i16> [[TMP34]], i16 [[TMP31]], i32 3 +; UNROLL-NO-IC-NEXT: [[TMP35]] = insertelement <4 x i16> [[TMP34]], i16 [[VECTOR_RECUR_EXTRACT]], i32 3 ; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP27]], <4 x i32> ; UNROLL-NO-IC-NEXT: [[TMP37:%.*]] = shufflevector <4 x i16> [[TMP27]], <4 x i16> [[TMP35]], <4 x i32> ; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = sext <4 x i16> [[TMP36]] to <4 x i32> @@ -2066,7 +2062,6 @@ define void @PR34711(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %n) { ; UNROLL-NO-IC-NEXT: br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] ; UNROLL-NO-IC: middle.block: ; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] -; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP35]], i32 3 ; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; UNROLL-NO-IC: scalar.ph: ; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -2177,11 +2172,11 @@ define void @PR34711(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %n) { ; SINK-AFTER-NEXT: [[TMP10:%.*]] = load i16, ptr [[TMP5]], align 2 ; SINK-AFTER-NEXT: [[TMP11:%.*]] = load i16, ptr [[TMP6]], align 2 ; SINK-AFTER-NEXT: [[TMP12:%.*]] = load i16, ptr [[TMP7]], align 2 -; SINK-AFTER-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP8]], align 2 +; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = load i16, ptr [[TMP8]], align 2 ; SINK-AFTER-NEXT: [[TMP14:%.*]] = insertelement <4 x i16> poison, i16 [[TMP10]], i32 0 ; SINK-AFTER-NEXT: [[TMP15:%.*]] = insertelement <4 x i16> [[TMP14]], i16 [[TMP11]], i32 1 ; SINK-AFTER-NEXT: [[TMP16:%.*]] = insertelement <4 x i16> [[TMP15]], i16 [[TMP12]], i32 2 -; SINK-AFTER-NEXT: [[TMP17]] = insertelement <4 x i16> [[TMP16]], i16 [[TMP13]], i32 3 +; SINK-AFTER-NEXT: [[TMP17]] = insertelement <4 x i16> [[TMP16]], i16 [[VECTOR_RECUR_EXTRACT]], i32 3 ; SINK-AFTER-NEXT: [[TMP18:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP17]], <4 x i32> ; SINK-AFTER-NEXT: [[TMP19:%.*]] = sext <4 x i16> [[TMP18]] to <4 x i32> ; SINK-AFTER-NEXT: [[TMP20:%.*]] = sext <4 x i16> [[TMP17]] to <4 x i32> @@ -2194,7 +2189,6 @@ define void @PR34711(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %n) { ; SINK-AFTER-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] ; SINK-AFTER: middle.block: ; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] -; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP17]], i32 3 ; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; SINK-AFTER: scalar.ph: ; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -2294,8 +2288,8 @@ define void @sink_after_with_multiple_users(ptr noalias %a, ptr noalias %b, i64 ; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] ; UNROLL-NO-IC: middle.block: -; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i32 3 +; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; UNROLL-NO-IC: scalar.ph: ; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -2407,8 +2401,8 @@ define void @sink_after_with_multiple_users(ptr noalias %a, ptr noalias %b, i64 ; SINK-AFTER-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SINK-AFTER-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] ; SINK-AFTER: middle.block: -; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; SINK-AFTER-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 +; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; SINK-AFTER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; SINK-AFTER: scalar.ph: ; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll index 50a5cc6774c5..df61798853ef 100644 --- a/llvm/test/Transforms/LoopVectorize/induction.ll +++ b/llvm/test/Transforms/LoopVectorize/induction.ll @@ -6256,8 +6256,8 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n ; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[VEC_IND]], i32 1 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -6327,8 +6327,8 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n ; IND-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; IND-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]] ; IND: middle.block: -; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; IND-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[VEC_IND]], i64 1 +; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; IND-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; IND: scalar.ph: ; IND-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -6402,8 +6402,8 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n ; UNROLL-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; UNROLL-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]] ; UNROLL: middle.block: -; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[STEP_ADD]], i64 1 +; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; UNROLL-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; UNROLL: scalar.ph: ; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -6485,8 +6485,8 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n ; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]] ; UNROLL-NO-IC: middle.block: -; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[STEP_ADD]], i32 1 +; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; UNROLL-NO-IC: scalar.ph: ; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] @@ -6560,8 +6560,8 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n ; INTERLEAVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; INTERLEAVE-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]] ; INTERLEAVE: middle.block: -; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; INTERLEAVE-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STEP_ADD]], i64 3 +; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; INTERLEAVE: scalar.ph: ; INTERLEAVE-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll index 078d6ca35ba1..bcc8632da4c6 100644 --- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -202,7 +202,10 @@ exit: ; DBG-NEXT: Successor(s): middle.block ; DBG-EMPTY: ; DBG-NEXT: middle.block: +; DBG-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end vp<[[SCALAR_STEPS]]>, ir<1> ; DBG-NEXT: No successors +; DBG-EMPTY: +; DBG-NEXT: Live-out i32 %for = vp<[[RESUME_1]]> ; DBG-NEXT: } define void @first_order_recurrence_using_induction(i32 %n, ptr %dst) { diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll index 4c3377255b21..0f3e7e6ac401 100644 --- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll @@ -1511,8 +1511,8 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]] ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <8 x i16> [[WIDE_VEC]], i64 7 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ] diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index 4a71e18ea377..4c93cb870fad 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -912,9 +912,11 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) { ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: ; CHECK-NEXT: EMIT vp<[[FOR_RESULT:%.+]]> = extract-from-end ir<%for.1.next>, ir<2> +; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%for.1.next>, ir<1> ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out i16 %for.1.lcssa = vp<[[FOR_RESULT]]> +; CHECK-NEXT: Live-out i16 %for.1 = vp<[[RESUME_1]]> ; CHECK-NEXT: } ; entry: -- GitLab From 2c31bc7c0455e2167dcaef6b284cb0574406fc72 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Wed, 5 Jun 2024 11:26:34 +0100 Subject: [PATCH 005/675] [flang][Semantics][OpenMP] Fix ICE for unknown reduction starting with . (#94398) In this case the union inside of the `parser::DefinedOperator` contains a string name instead of the expected `parser::DefinedOperator::IntrinsicOperator`. This led to a `std::abort`. This patch adapts the code so that if it contains a string name we emit a semantic error. --- flang/lib/Semantics/check-omp-structure.cpp | 12 +++++++++--- flang/test/Semantics/OpenMP/reduction13.f90 | 10 ++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 flang/test/Semantics/OpenMP/reduction13.f90 diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp index 5e3a5725c18d..54ce45157537 100644 --- a/flang/lib/Semantics/check-omp-structure.cpp +++ b/flang/lib/Semantics/check-omp-structure.cpp @@ -2321,9 +2321,15 @@ bool OmpStructureChecker::CheckReductionOperators( common::visit( common::visitors{ [&](const parser::DefinedOperator &dOpr) { - const auto &intrinsicOp{ - std::get(dOpr.u)}; - ok = CheckIntrinsicOperator(intrinsicOp); + if (const auto *intrinsicOp{ + std::get_if( + &dOpr.u)}) { + ok = CheckIntrinsicOperator(*intrinsicOp); + } else { + context_.Say(GetContext().clauseSource, + "Invalid reduction operator in REDUCTION clause."_err_en_US, + ContextDirectiveAsFortran()); + } }, [&](const parser::ProcedureDesignator &procD) { const parser::Name *name{std::get_if(&procD.u)}; diff --git a/flang/test/Semantics/OpenMP/reduction13.f90 b/flang/test/Semantics/OpenMP/reduction13.f90 new file mode 100644 index 000000000000..edb153ce20ef --- /dev/null +++ b/flang/test/Semantics/OpenMP/reduction13.f90 @@ -0,0 +1,10 @@ +! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp +! OpenMP Version 4.5 +! 2.15.3.6 Reduction Clause +program omp_reduction + integer :: k + ! misspelling. Should be "min" + !ERROR: Invalid reduction operator in REDUCTION clause. + !$omp parallel reduction(.min.:k) + !$omp end parallel +end program omp_reduction -- GitLab From 29a925abb660104b413b15075b3a19793825f57e Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell Date: Wed, 5 Jun 2024 11:35:13 +0100 Subject: [PATCH 006/675] [mlir][affine][Analysis] Add conservative bounds for semi-affine mods (#93576) This patch adds support for computing bounds for semi-affine mod expression to FlatLinearConstraints. This is then enabled within the ScalableValueBoundsConstraintSet to allow computing the bounds of scalable remainder loops. E.g. computing the bound of something like: ``` // `1000 mod s0` is a semi-affine. #remainder_start_index = affine_map<()[s0] -> (-(1000 mod s0) + 1000)> #remaining_iterations = affine_map<(d0) -> (-d0 + 1000)> %0 = affine.apply #remainder_start_index()[%c8_vscale] scf.for %i = %0 to %c1000 step %c8_vscale { %remaining_iterations = affine.apply #remaining_iterations(%i) // The upper bound for the remainder loop iterations should be: // %c8_vscale - 1 (expressed as an affine map, // affine_map<()[s0] -> (s0 * 8 - 1)>, where s0 is vscale) %bound = "test.reify_bound"(%remaining_iterations) <{scalable, ...}> } ``` There are caveats to this implementation. To be able to add a bound for a `mod` we need to assume the rhs is positive (> 0). This may not be known when adding the bounds for the `mod` expression. So to handle this a constraint is added for `rhs > 0`, this may later be found not to hold (in which case the constraints set becomes empty/invalid). This is not a problem for computing scalable bounds where it's safe to assume `s0` is vscale (or some positive multiple of it). But this may need to be considered when enabling this feature elsewhere (to ensure correctness). --- .../Analysis/FlatLinearValueConstraints.h | 49 +++-- .../IR/ScalableValueBoundsConstraintSet.h | 5 +- mlir/include/mlir/IR/AffineExprVisitor.h | 16 +- .../mlir/Interfaces/ValueBoundsOpInterface.h | 6 +- .../Analysis/FlatLinearValueConstraints.cpp | 169 +++++++++++++----- .../IR/ScalableValueBoundsConstraintSet.cpp | 12 +- mlir/lib/IR/AffineExpr.cpp | 30 ++-- .../lib/Interfaces/ValueBoundsOpInterface.cpp | 18 +- .../Dialect/Vector/test-scalable-bounds.mlir | 56 ++++++ 9 files changed, 275 insertions(+), 86 deletions(-) diff --git a/mlir/include/mlir/Analysis/FlatLinearValueConstraints.h b/mlir/include/mlir/Analysis/FlatLinearValueConstraints.h index 29c19442a7c7..cc6ab64b4b7d 100644 --- a/mlir/include/mlir/Analysis/FlatLinearValueConstraints.h +++ b/mlir/include/mlir/Analysis/FlatLinearValueConstraints.h @@ -66,6 +66,10 @@ public: /// Return the kind of this object. Kind getKind() const override { return Kind::FlatLinearConstraints; } + /// Flag to control if conservative semi-affine bounds should be added in + /// `addBound()`. + enum class AddConservativeSemiAffineBounds { No = 0, Yes }; + /// Adds a bound for the variable at the specified position with constraints /// being drawn from the specified bound map. In case of an EQ bound, the /// bound map is expected to have exactly one result. In case of a LB/UB, the @@ -77,21 +81,39 @@ public: /// as a closed bound by +1/-1 respectively. In case of an EQ bound, it can /// only be added as a closed bound. /// + /// Conservative bounds for semi-affine expressions will be added if + /// `AddConservativeSemiAffineBounds` is set to `Yes`. This currently only + /// covers semi-affine `mod` expressions, so `addBound()` will still fail if + /// it encounters a semi-affine `floordiv`, `ceildiv`, or `mul`. Note: If + /// enabled it is possible for the resulting constraint set to become empty if + /// a precondition of a conservative bound is found not to hold. + /// /// Note: The dimensions/symbols of this FlatLinearConstraints must match the /// dimensions/symbols of the affine map. - LogicalResult addBound(presburger::BoundType type, unsigned pos, - AffineMap boundMap, bool isClosedBound); + LogicalResult addBound( + presburger::BoundType type, unsigned pos, AffineMap boundMap, + bool isClosedBound, + AddConservativeSemiAffineBounds = AddConservativeSemiAffineBounds::No); /// Adds a bound for the variable at the specified position with constraints /// being drawn from the specified bound map. In case of an EQ bound, the /// bound map is expected to have exactly one result. In case of a LB/UB, the /// bound map may have more than one result, for each of which an inequality /// is added. + /// + /// Conservative bounds for semi-affine expressions will be added if + /// `AddConservativeSemiAffineBounds` is set to `Yes`. This currently only + /// covers semi-affine `mod` expressions, so `addBound()` will still fail if + /// it encounters a semi-affine `floordiv`, `ceildiv`, or `mul`. Note: If + /// enabled it is possible for the resulting constraint set to become empty if + /// a precondition of a conservative bound is found not to hold. + /// /// Note: The dimensions/symbols of this FlatLinearConstraints must match the /// dimensions/symbols of the affine map. By default the lower bound is closed /// and the upper bound is open. - LogicalResult addBound(presburger::BoundType type, unsigned pos, - AffineMap boundMap); + LogicalResult addBound( + presburger::BoundType type, unsigned pos, AffineMap boundMap, + AddConservativeSemiAffineBounds = AddConservativeSemiAffineBounds::No); /// The `addBound` overload above hides the inherited overloads by default, so /// we explicitly introduce them here. @@ -193,7 +215,8 @@ protected: /// Note: This is a shared helper function of `addLowerOrUpperBound` and /// `composeMatchingMap`. LogicalResult flattenAlignedMapAndMergeLocals( - AffineMap map, std::vector> *flattenedExprs); + AffineMap map, std::vector> *flattenedExprs, + bool addConservativeSemiAffineBounds = false); /// Prints the number of constraints, dimensions, symbols and locals in the /// FlatLinearConstraints. Also, prints for each variable whether there is @@ -468,18 +491,19 @@ public: /// Flattens 'expr' into 'flattenedExpr', which contains the coefficients of the /// dimensions, symbols, and additional variables that represent floor divisions /// of dimensions, symbols, and in turn other floor divisions. Returns failure -/// if 'expr' could not be flattened (i.e., semi-affine is not yet handled). +/// if 'expr' could not be flattened (i.e., an unhandled semi-affine was found). /// 'cst' contains constraints that connect newly introduced local variables /// to existing dimensional and symbolic variables. See documentation for /// AffineExprFlattener on how mod's and div's are flattened. -LogicalResult getFlattenedAffineExpr(AffineExpr expr, unsigned numDims, - unsigned numSymbols, - SmallVectorImpl *flattenedExpr, - FlatLinearConstraints *cst = nullptr); +LogicalResult +getFlattenedAffineExpr(AffineExpr expr, unsigned numDims, unsigned numSymbols, + SmallVectorImpl *flattenedExpr, + FlatLinearConstraints *cst = nullptr, + bool addConservativeSemiAffineBounds = false); /// Flattens the result expressions of the map to their corresponding flattened /// forms and set in 'flattenedExprs'. Returns failure if any expression in the -/// map could not be flattened (i.e., semi-affine is not yet handled). 'cst' +/// map could not be flattened (i.e., an unhandled semi-affine was found). 'cst' /// contains constraints that connect newly introduced local variables to /// existing dimensional and / symbolic variables. See documentation for /// AffineExprFlattener on how mod's and div's are flattened. For all affine @@ -490,7 +514,8 @@ LogicalResult getFlattenedAffineExpr(AffineExpr expr, unsigned numDims, LogicalResult getFlattenedAffineExprs(AffineMap map, std::vector> *flattenedExprs, - FlatLinearConstraints *cst = nullptr); + FlatLinearConstraints *cst = nullptr, + bool addConservativeSemiAffineBounds = false); LogicalResult getFlattenedAffineExprs(IntegerSet set, std::vector> *flattenedExprs, diff --git a/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h b/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h index 67a6581eb2fb..93b3c92533c5 100644 --- a/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h +++ b/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h @@ -33,8 +33,9 @@ struct ScalableValueBoundsConstraintSet MLIRContext *context, ValueBoundsConstraintSet::StopConditionFn stopCondition, unsigned vscaleMin, unsigned vscaleMax) - : RTTIExtends(context, stopCondition), vscaleMin(vscaleMin), - vscaleMax(vscaleMax) {}; + : RTTIExtends(context, stopCondition, + /*addConservativeSemiAffineBounds=*/true), + vscaleMin(vscaleMin), vscaleMax(vscaleMax) {}; using RTTIExtends::bound; using RTTIExtends::StopConditionFn; diff --git a/mlir/include/mlir/IR/AffineExprVisitor.h b/mlir/include/mlir/IR/AffineExprVisitor.h index 27c49cd80018..fc4cd915d845 100644 --- a/mlir/include/mlir/IR/AffineExprVisitor.h +++ b/mlir/include/mlir/IR/AffineExprVisitor.h @@ -413,18 +413,22 @@ protected: /// lhs of the mod, floordiv, ceildiv or mul expression and with respect to a /// symbolic rhs expression. `localExpr` is the simplified tree expression /// (AffineExpr) corresponding to the quantifier. - virtual void addLocalIdSemiAffine(AffineExpr localExpr); + virtual LogicalResult addLocalIdSemiAffine(ArrayRef lhs, + ArrayRef rhs, + AffineExpr localExpr); private: - /// Adds `expr`, which may be mod, ceildiv, floordiv or mod expression + /// Adds `localExpr`, which may be mod, ceildiv, floordiv or mod expression /// representing the affine expression corresponding to the quantifier - /// introduced as the local variable corresponding to `expr`. If the + /// introduced as the local variable corresponding to `localExpr`. If the /// quantifier is already present, we put the coefficient in the proper index /// of `result`, otherwise we add a new local variable and put the coefficient /// there. - void addLocalVariableSemiAffine(AffineExpr expr, - SmallVectorImpl &result, - unsigned long resultSize); + LogicalResult addLocalVariableSemiAffine(ArrayRef lhs, + ArrayRef rhs, + AffineExpr localExpr, + SmallVectorImpl &result, + unsigned long resultSize); // t = expr floordiv c <=> t = q, c * q <= expr <= c * q + c - 1 // A floordiv is thus flattened by introducing a new local variable q, and diff --git a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h index ac17ace5a976..337314143c80 100644 --- a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h +++ b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h @@ -313,7 +313,8 @@ protected: /// An index-typed value or the dimension of a shaped-type value. using ValueDim = std::pair; - ValueBoundsConstraintSet(MLIRContext *ctx, StopConditionFn stopCondition); + ValueBoundsConstraintSet(MLIRContext *ctx, StopConditionFn stopCondition, + bool addConservativeSemiAffineBounds = false); /// Return "true" if, based on the current state of the constraint system, /// "lhs cmp rhs" was proven to hold. Return "false" if the specified relation @@ -404,6 +405,9 @@ protected: /// The current stop condition function. StopConditionFn stopCondition = nullptr; + + /// Should conservative bounds be added for semi-affine expressions. + bool addConservativeSemiAffineBounds = false; }; } // namespace mlir diff --git a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp index 8b38016d6149..bf7e3121ea32 100644 --- a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp +++ b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp @@ -36,11 +36,12 @@ using namespace presburger; namespace { // See comments for SimpleAffineExprFlattener. -// An AffineExprFlattener extends a SimpleAffineExprFlattener by recording -// constraint information associated with mod's, floordiv's, and ceildiv's -// in FlatLinearConstraints 'localVarCst'. +// An AffineExprFlattenerWithLocalVars extends a SimpleAffineExprFlattener by +// recording constraint information associated with mod's, floordiv's, and +// ceildiv's in FlatLinearConstraints 'localVarCst'. struct AffineExprFlattener : public SimpleAffineExprFlattener { -public: + using SimpleAffineExprFlattener::SimpleAffineExprFlattener; + // Constraints connecting newly introduced local variables (for mod's and // div's) to existing (dimensional and symbolic) ones. These are always // inequalities. @@ -48,7 +49,7 @@ public: AffineExprFlattener(unsigned nDims, unsigned nSymbols) : SimpleAffineExprFlattener(nDims, nSymbols), - localVarCst(PresburgerSpace::getSetSpace(nDims, nSymbols)) {} + localVarCst(PresburgerSpace::getSetSpace(nDims, nSymbols)) {}; private: // Add a local variable (needed to flatten a mod, floordiv, ceildiv expr). @@ -63,76 +64,144 @@ private: // Update localVarCst. localVarCst.addLocalFloorDiv(dividend, divisor); } + + LogicalResult addLocalIdSemiAffine(ArrayRef lhs, + ArrayRef rhs, + AffineExpr localExpr) override { + // AffineExprFlattener does not support semi-affine expressions. + return failure(); + } +}; + +// A SemiAffineExprFlattener is an AffineExprFlattenerWithLocalVars that adds +// conservative bounds for semi-affine expressions (given assumptions hold). If +// the assumptions required to add the semi-affine bounds are found not to hold +// the final constraints set will be empty/inconsistent. If the assumptions are +// never contradicted the final bounds still only will be correct if the +// assumptions hold. +struct SemiAffineExprFlattener : public AffineExprFlattener { + using AffineExprFlattener::AffineExprFlattener; + + LogicalResult addLocalIdSemiAffine(ArrayRef lhs, + ArrayRef rhs, + AffineExpr localExpr) override { + auto result = + SimpleAffineExprFlattener::addLocalIdSemiAffine(lhs, rhs, localExpr); + assert(succeeded(result) && + "unexpected failure in SimpleAffineExprFlattener"); + (void)result; + + if (localExpr.getKind() == AffineExprKind::Mod) { + // Given two numbers a and b, division is defined as: + // + // a = bq + r + // 0 <= r < |b| (where |x| is the absolute value of x) + // + // q = a floordiv b + // r = a mod b + + // Add a new local variable (r) to represent the mod. + unsigned rPos = localVarCst.appendVar(VarKind::Local); + + // r >= 0 (Can ALWAYS be added) + localVarCst.addBound(BoundType::LB, rPos, 0); + + // r < b (Can be added if b > 0, which we assume here) + ArrayRef b = rhs; + SmallVector bSubR(b); + bSubR.insert(bSubR.begin() + rPos, -1); + // Note: bSubR = b - r + // So this adds the bound b - r >= 1 (equivalent to r < b) + localVarCst.addBound(BoundType::LB, bSubR, 1); + + // Note: The assumption of b > 0 is based on the affine expression docs, + // which state "RHS of mod is always a constant or a symbolic expression + // with a positive value." (see AffineExprKind in AffineExpr.h). If this + // assumption does not hold constraints (added above) are a contradiction. + + return success(); + } + + // TODO: Support other semi-affine expressions. + return failure(); + } }; } // namespace // Flattens the expressions in map. Returns failure if 'expr' was unable to be // flattened. For example two specific cases: -// 1. semi-affine expressions not handled yet. +// 1. an unhandled semi-affine expressions is found. // 2. has poison expression (i.e., division by zero). static LogicalResult getFlattenedAffineExprs(ArrayRef exprs, unsigned numDims, unsigned numSymbols, std::vector> *flattenedExprs, - FlatLinearConstraints *localVarCst) { + FlatLinearConstraints *localVarCst, + bool addConservativeSemiAffineBounds = false) { if (exprs.empty()) { if (localVarCst) *localVarCst = FlatLinearConstraints(numDims, numSymbols); return success(); } - AffineExprFlattener flattener(numDims, numSymbols); - // Use the same flattener to simplify each expression successively. This way - // local variables / expressions are shared. - for (auto expr : exprs) { - if (!expr.isPureAffine()) - return failure(); - // has poison expression - auto flattenResult = flattener.walkPostOrder(expr); - if (failed(flattenResult)) - return failure(); - } + auto flattenExprs = [&](AffineExprFlattener &flattener) -> LogicalResult { + // Use the same flattener to simplify each expression successively. This way + // local variables / expressions are shared. + for (auto expr : exprs) { + auto flattenResult = flattener.walkPostOrder(expr); + if (failed(flattenResult)) + return failure(); + } - assert(flattener.operandExprStack.size() == exprs.size()); - flattenedExprs->clear(); - flattenedExprs->assign(flattener.operandExprStack.begin(), - flattener.operandExprStack.end()); + assert(flattener.operandExprStack.size() == exprs.size()); + flattenedExprs->clear(); + flattenedExprs->assign(flattener.operandExprStack.begin(), + flattener.operandExprStack.end()); - if (localVarCst) - localVarCst->clearAndCopyFrom(flattener.localVarCst); + if (localVarCst) + localVarCst->clearAndCopyFrom(flattener.localVarCst); - return success(); + return success(); + }; + + if (addConservativeSemiAffineBounds) { + SemiAffineExprFlattener flattener(numDims, numSymbols); + return flattenExprs(flattener); + } + + AffineExprFlattener flattener(numDims, numSymbols); + return flattenExprs(flattener); } // Flattens 'expr' into 'flattenedExpr'. Returns failure if 'expr' was unable to -// be flattened (semi-affine expressions not handled yet). -LogicalResult -mlir::getFlattenedAffineExpr(AffineExpr expr, unsigned numDims, - unsigned numSymbols, - SmallVectorImpl *flattenedExpr, - FlatLinearConstraints *localVarCst) { +// be flattened (an unhandled semi-affine was found). +LogicalResult mlir::getFlattenedAffineExpr( + AffineExpr expr, unsigned numDims, unsigned numSymbols, + SmallVectorImpl *flattenedExpr, FlatLinearConstraints *localVarCst, + bool addConservativeSemiAffineBounds) { std::vector> flattenedExprs; - LogicalResult ret = ::getFlattenedAffineExprs({expr}, numDims, numSymbols, - &flattenedExprs, localVarCst); + LogicalResult ret = + ::getFlattenedAffineExprs({expr}, numDims, numSymbols, &flattenedExprs, + localVarCst, addConservativeSemiAffineBounds); *flattenedExpr = flattenedExprs[0]; return ret; } /// Flattens the expressions in map. Returns failure if 'expr' was unable to be -/// flattened (i.e., semi-affine expressions not handled yet). +/// flattened (i.e., an unhandled semi-affine was found). LogicalResult mlir::getFlattenedAffineExprs( AffineMap map, std::vector> *flattenedExprs, - FlatLinearConstraints *localVarCst) { + FlatLinearConstraints *localVarCst, bool addConservativeSemiAffineBounds) { if (map.getNumResults() == 0) { if (localVarCst) *localVarCst = FlatLinearConstraints(map.getNumDims(), map.getNumSymbols()); return success(); } - return ::getFlattenedAffineExprs(map.getResults(), map.getNumDims(), - map.getNumSymbols(), flattenedExprs, - localVarCst); + return ::getFlattenedAffineExprs( + map.getResults(), map.getNumDims(), map.getNumSymbols(), flattenedExprs, + localVarCst, addConservativeSemiAffineBounds); } LogicalResult mlir::getFlattenedAffineExprs( @@ -641,9 +710,11 @@ void FlatLinearConstraints::getSliceBounds(unsigned offset, unsigned num, } LogicalResult FlatLinearConstraints::flattenAlignedMapAndMergeLocals( - AffineMap map, std::vector> *flattenedExprs) { + AffineMap map, std::vector> *flattenedExprs, + bool addConservativeSemiAffineBounds) { FlatLinearConstraints localCst; - if (failed(getFlattenedAffineExprs(map, flattenedExprs, &localCst))) { + if (failed(getFlattenedAffineExprs(map, flattenedExprs, &localCst, + addConservativeSemiAffineBounds))) { LLVM_DEBUG(llvm::dbgs() << "composition unimplemented for semi-affine maps\n"); return failure(); @@ -664,9 +735,9 @@ LogicalResult FlatLinearConstraints::flattenAlignedMapAndMergeLocals( return success(); } -LogicalResult FlatLinearConstraints::addBound(BoundType type, unsigned pos, - AffineMap boundMap, - bool isClosedBound) { +LogicalResult FlatLinearConstraints::addBound( + BoundType type, unsigned pos, AffineMap boundMap, bool isClosedBound, + AddConservativeSemiAffineBounds addSemiAffineBounds) { assert(boundMap.getNumDims() == getNumDimVars() && "dim mismatch"); assert(boundMap.getNumSymbols() == getNumSymbolVars() && "symbol mismatch"); assert(pos < getNumDimAndSymbolVars() && "invalid position"); @@ -680,7 +751,9 @@ LogicalResult FlatLinearConstraints::addBound(BoundType type, unsigned pos, bool lower = type == BoundType::LB || type == BoundType::EQ; std::vector> flatExprs; - if (failed(flattenAlignedMapAndMergeLocals(boundMap, &flatExprs))) + if (failed(flattenAlignedMapAndMergeLocals( + boundMap, &flatExprs, + addSemiAffineBounds == AddConservativeSemiAffineBounds::Yes))) return failure(); assert(flatExprs.size() == boundMap.getNumResults()); @@ -716,9 +789,11 @@ LogicalResult FlatLinearConstraints::addBound(BoundType type, unsigned pos, return success(); } -LogicalResult FlatLinearConstraints::addBound(BoundType type, unsigned pos, - AffineMap boundMap) { - return addBound(type, pos, boundMap, /*isClosedBound=*/type != BoundType::UB); +LogicalResult FlatLinearConstraints::addBound( + BoundType type, unsigned pos, AffineMap boundMap, + AddConservativeSemiAffineBounds addSemiAffineBounds) { + return addBound(type, pos, boundMap, + /*isClosedBound=*/type != BoundType::UB, addSemiAffineBounds); } /// Compute an explicit representation for local vars. For all systems coming diff --git a/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp b/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp index f8df34843a36..9c365376c84c 100644 --- a/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp +++ b/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp @@ -7,9 +7,7 @@ //===----------------------------------------------------------------------===// #include "mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h" - #include "mlir/Dialect/Vector/IR/VectorOps.h" - namespace mlir::vector { FailureOr @@ -62,6 +60,11 @@ ScalableValueBoundsConstraintSet::computeScalableBound( int64_t pos = scalableCstr.insert(value, dim, /*isSymbol=*/false); scalableCstr.processWorklist(); + // Check the resulting constraints set is valid. + if (scalableCstr.cstr.isEmpty()) { + return failure(); + } + // Project out all columns apart from vscale and the starting point // (value/dim). This should result in constraints in terms of vscale only. auto projectOutFn = [&](ValueDim p) { @@ -71,6 +74,11 @@ ScalableValueBoundsConstraintSet::computeScalableBound( return p.first != scalableCstr.getVscaleValue() && !isStartingPoint; }; scalableCstr.projectOut(projectOutFn); + // Also project out local variables (these are not tracked by the + // ValueBoundsConstraintSet). + for (unsigned i = 0, e = scalableCstr.cstr.getNumLocalVars(); i < e; ++i) { + scalableCstr.cstr.projectOut(scalableCstr.cstr.getNumDimAndSymbolVars()); + } assert(scalableCstr.cstr.getNumDimAndSymbolVars() == scalableCstr.positionToValueDim.size() && diff --git a/mlir/lib/IR/AffineExpr.cpp b/mlir/lib/IR/AffineExpr.cpp index 94562d0f15a2..5f2016470b25 100644 --- a/mlir/lib/IR/AffineExpr.cpp +++ b/mlir/lib/IR/AffineExpr.cpp @@ -1242,13 +1242,13 @@ LogicalResult SimpleAffineExprFlattener::visitMulExpr(AffineBinaryOpExpr expr) { // variable in place of the product; the affine expression // corresponding to the quantifier is added to `localExprs`. if (!isa(expr.getRHS())) { + SmallVector mulLhs(lhs); MLIRContext *context = expr.getContext(); AffineExpr a = getAffineExprFromFlatForm(lhs, numDims, numSymbols, localExprs, context); AffineExpr b = getAffineExprFromFlatForm(rhs, numDims, numSymbols, localExprs, context); - addLocalVariableSemiAffine(a * b, lhs, lhs.size()); - return success(); + return addLocalVariableSemiAffine(mulLhs, rhs, a * b, lhs, lhs.size()); } // Get the RHS constant. @@ -1295,13 +1295,13 @@ LogicalResult SimpleAffineExprFlattener::visitModExpr(AffineBinaryOpExpr expr) { // variable in place of the modulo value, and the affine expression // corresponding to the quantifier is added to `localExprs`. if (!isa(expr.getRHS())) { + SmallVector modLhs(lhs); AffineExpr dividendExpr = getAffineExprFromFlatForm( lhs, numDims, numSymbols, localExprs, context); AffineExpr divisorExpr = getAffineExprFromFlatForm(rhs, numDims, numSymbols, localExprs, context); AffineExpr modExpr = dividendExpr % divisorExpr; - addLocalVariableSemiAffine(modExpr, lhs, lhs.size()); - return success(); + return addLocalVariableSemiAffine(modLhs, rhs, modExpr, lhs, lhs.size()); } int64_t rhsConst = rhs[getConstantIndex()]; @@ -1385,19 +1385,22 @@ SimpleAffineExprFlattener::visitConstantExpr(AffineConstantExpr expr) { return success(); } -void SimpleAffineExprFlattener::addLocalVariableSemiAffine( - AffineExpr expr, SmallVectorImpl &result, - unsigned long resultSize) { +LogicalResult SimpleAffineExprFlattener::addLocalVariableSemiAffine( + ArrayRef lhs, ArrayRef rhs, AffineExpr localExpr, + SmallVectorImpl &result, unsigned long resultSize) { assert(result.size() == resultSize && "`result` vector passed is not of correct size"); int loc; - if ((loc = findLocalId(expr)) == -1) - addLocalIdSemiAffine(expr); + if ((loc = findLocalId(localExpr)) == -1) { + if (failed(addLocalIdSemiAffine(lhs, rhs, localExpr))) + return failure(); + } std::fill(result.begin(), result.end(), 0); if (loc == -1) result[getLocalVarStartIndex() + numLocals - 1] = 1; else result[getLocalVarStartIndex() + loc] = 1; + return success(); } // t = expr floordiv c <=> t = q, c * q <= expr <= c * q + c - 1 @@ -1426,13 +1429,13 @@ LogicalResult SimpleAffineExprFlattener::visitDivExpr(AffineBinaryOpExpr expr, // variable in place of the quotient, and the affine expression corresponding // to the quantifier is added to `localExprs`. if (!isa(expr.getRHS())) { + SmallVector divLhs(lhs); AffineExpr a = getAffineExprFromFlatForm(lhs, numDims, numSymbols, localExprs, context); AffineExpr b = getAffineExprFromFlatForm(rhs, numDims, numSymbols, localExprs, context); AffineExpr divExpr = isCeil ? a.ceilDiv(b) : a.floorDiv(b); - addLocalVariableSemiAffine(divExpr, lhs, lhs.size()); - return success(); + return addLocalVariableSemiAffine(divLhs, rhs, divExpr, lhs, lhs.size()); } // This is a pure affine expr; the RHS is a positive constant. @@ -1503,11 +1506,14 @@ void SimpleAffineExprFlattener::addLocalFloorDivId(ArrayRef dividend, // dividend and divisor are not used here; an override of this method uses it. } -void SimpleAffineExprFlattener::addLocalIdSemiAffine(AffineExpr localExpr) { +LogicalResult SimpleAffineExprFlattener::addLocalIdSemiAffine( + ArrayRef lhs, ArrayRef rhs, AffineExpr localExpr) { for (SmallVector &subExpr : operandExprStack) subExpr.insert(subExpr.begin() + getLocalVarStartIndex() + numLocals, 0); localExprs.push_back(localExpr); ++numLocals; + // lhs and rhs are not used here; an override of this method uses them. + return success(); } int SimpleAffineExprFlattener::findLocalId(AffineExpr localExpr) { diff --git a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp index 87937591e60a..6420c192b257 100644 --- a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp +++ b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp @@ -151,8 +151,10 @@ ValueBoundsConstraintSet::Variable::Variable(AffineMap map, [](Value v) { return Variable(v); })) {} ValueBoundsConstraintSet::ValueBoundsConstraintSet( - MLIRContext *ctx, StopConditionFn stopCondition) - : builder(ctx), stopCondition(stopCondition) { + MLIRContext *ctx, StopConditionFn stopCondition, + bool addConservativeSemiAffineBounds) + : builder(ctx), stopCondition(stopCondition), + addConservativeSemiAffineBounds(addConservativeSemiAffineBounds) { assert(stopCondition && "expected non-null stop condition"); } @@ -174,11 +176,19 @@ static void assertValidValueDim(Value value, std::optional dim) { void ValueBoundsConstraintSet::addBound(BoundType type, int64_t pos, AffineExpr expr) { + // Note: If `addConservativeSemiAffineBounds` is true then the bound + // computation function needs to handle the case that the constraints set + // could become empty. This is because the conservative bounds add assumptions + // (e.g. for `mod` it assumes `rhs > 0`). If these constraints are later found + // not to hold, then the bound is invalid. LogicalResult status = cstr.addBound( type, pos, - AffineMap::get(cstr.getNumDimVars(), cstr.getNumSymbolVars(), expr)); + AffineMap::get(cstr.getNumDimVars(), cstr.getNumSymbolVars(), expr), + addConservativeSemiAffineBounds + ? FlatLinearConstraints::AddConservativeSemiAffineBounds::Yes + : FlatLinearConstraints::AddConservativeSemiAffineBounds::No); if (failed(status)) { - // Non-pure (e.g., semi-affine) expressions are not yet supported by + // Not all semi-affine expressions are not yet supported by // FlatLinearConstraints. However, we can just ignore such failures here. // Even without this bound, there may be enough information in the // constraint system to compute the requested bound. In case this bound is diff --git a/mlir/test/Dialect/Vector/test-scalable-bounds.mlir b/mlir/test/Dialect/Vector/test-scalable-bounds.mlir index d549c5bd1c37..673e03f05c1b 100644 --- a/mlir/test/Dialect/Vector/test-scalable-bounds.mlir +++ b/mlir/test/Dialect/Vector/test-scalable-bounds.mlir @@ -159,3 +159,59 @@ func.func @non_scalable_code() { } return } + +// ----- + +#remainder_start_index = affine_map<()[s0] -> (-(1000 mod s0) + 1000)> +#remaining_iterations = affine_map<(d0) -> (-d0 + 1000)> + +// CHECK: #[[$REMAINDER_START_MAP:.*]] = affine_map<()[s0] -> (-(1000 mod s0) + 1000)> +// CHECK: #[[$SCALABLE_BOUND_MAP_4:.*]] = affine_map<()[s0] -> (s0 * 8 - 1)> + +// CHECK-LABEL: @test_scalable_remainder_loop +// CHECK: %[[VSCALE:.*]] = vector.vscale +// CHECK: %[[SCALABLE_BOUND:.*]] = affine.apply #[[$SCALABLE_BOUND_MAP_4]]()[%[[VSCALE]]] +// CHECK: "test.some_use"(%[[SCALABLE_BOUND]]) : (index) -> () +func.func @test_scalable_remainder_loop() { + %c8 = arith.constant 8 : index + %c1000 = arith.constant 1000 : index + %vscale = vector.vscale + %c8_vscale = arith.muli %vscale, %c8 : index + %0 = affine.apply #remainder_start_index()[%c8_vscale] + scf.for %arg1 = %0 to %c1000 step %c8_vscale { + %remaining_iterations = affine.apply #remaining_iterations(%arg1) + // The upper bound for the remainder loop iterations should be: %c8_vscale - 1 + // (expressed as an affine map, affine_map<()[s0] -> (s0 * 8 - 1)>, where s0 is vscale) + %bound = "test.reify_bound"(%remaining_iterations) <{scalable, type = "UB", vscale_min = 1 : i64, vscale_max = 16 : i64}> : (index) -> index + "test.some_use"(%bound) : (index) -> () + } + return +} + +// ----- + +#unsupported_semi_affine = affine_map<()[s0] -> (s0 * s0)> + +func.func @unsupported_semi_affine() { + %vscale = vector.vscale + %0 = affine.apply #unsupported_semi_affine()[%vscale] + // expected-error @below{{could not reify bound}} + %bound = "test.reify_bound"(%0) <{scalable, type = "UB", vscale_min = 1 : i64, vscale_max = 16 : i64}> : (index) -> index + "test.some_use"(%bound) : (index) -> () + return +} + +// ----- + +#map_mod = affine_map<()[s0] -> (1000 mod s0)> + +func.func @unsupported_negative_mod() { + %c_minus_1 = arith.constant -1 : index + %vscale = vector.vscale + %negative_vscale = arith.muli %vscale, %c_minus_1 : index + %0 = affine.apply #map_mod()[%negative_vscale] + // expected-error @below{{could not reify bound}} + %bound = "test.reify_bound"(%0) <{scalable, type = "UB", vscale_min = 1 : i64, vscale_max = 16 : i64}> : (index) -> index + "test.some_use"(%bound) : (index) -> () + return +} -- GitLab From 851710d7910609e176cda36e0d113274d6bd506d Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Wed, 5 Jun 2024 12:45:27 +0200 Subject: [PATCH 007/675] [SimplifyCFG] Add additional tests for sinking (NFC) This covers some interesting edge cases when the sink target is a loop header. --- .../SimplifyCFG/X86/sink-common-code.ll | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll index d24fc1f0142f..118372164c1f 100644 --- a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll +++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll @@ -1604,4 +1604,102 @@ if.end: ret void } +define void @loop_use_in_different_bb(i32 %n) { +; CHECK-LABEL: @loop_use_in_different_bb( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ADD:%.*]] = add i32 [[N:%.*]], 1 +; CHECK-NEXT: br label [[FOR_COND:%.*]] +; CHECK: for.cond: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ] +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV]], [[ADD]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[RETURN:%.*]], label [[FOR_BODY]] +; CHECK: for.body: +; CHECK-NEXT: [[INC]] = add i32 [[IV]], 1 +; CHECK-NEXT: br label [[FOR_COND]] +; CHECK: return: +; CHECK-NEXT: ret void +; +entry: + %add = add i32 %n, 1 + br label %for.cond + +for.cond: + %iv = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %exitcond = icmp eq i32 %iv, %add + br i1 %exitcond, label %return, label %for.body + +for.body: + %inc = add i32 %iv, 1 + br label %for.cond + +return: + ret void +} + +define void @loop_use_in_different_bb_phi(i32 %n) { +; CHECK-LABEL: @loop_use_in_different_bb_phi( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ADD:%.*]] = add i32 [[N:%.*]], 1 +; CHECK-NEXT: br label [[FOR_COND:%.*]] +; CHECK: for.cond: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ] +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV]], 42 +; CHECK-NEXT: br i1 [[EXITCOND]], label [[RETURN:%.*]], label [[FOR_BODY]] +; CHECK: for.body: +; CHECK-NEXT: [[DUMMY:%.*]] = phi i32 [ [[ADD]], [[FOR_COND]] ] +; CHECK-NEXT: [[INC]] = add i32 [[IV]], 1 +; CHECK-NEXT: br label [[FOR_COND]] +; CHECK: return: +; CHECK-NEXT: ret void +; +entry: + %add = add i32 %n, 1 + br label %for.cond + +for.cond: + %iv = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %exitcond = icmp eq i32 %iv, 42 + br i1 %exitcond, label %return, label %for.body + +for.body: + %dummy = phi i32 [ %add, %for.cond ] + %inc = add i32 %iv, 1 + br label %for.cond + +return: + ret void +} + +define void @loop_use_in_wrong_phi_operand(i32 %n) { +; CHECK-LABEL: @loop_use_in_wrong_phi_operand( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ADD:%.*]] = add i32 [[N:%.*]], 1 +; CHECK-NEXT: br label [[FOR_COND:%.*]] +; CHECK: for.cond: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY:%.*]] ] +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV]], 42 +; CHECK-NEXT: br i1 [[EXITCOND]], label [[RETURN:%.*]], label [[FOR_BODY]] +; CHECK: for.body: +; CHECK-NEXT: [[INC:%.*]] = add i32 [[IV]], 1 +; CHECK-NEXT: br label [[FOR_COND]] +; CHECK: return: +; CHECK-NEXT: ret void +; +entry: + %add = add i32 %n, 1 + br label %for.cond + +for.cond: + %iv = phi i32 [ 0, %entry ], [ %add, %for.body ] + %exitcond = icmp eq i32 %iv, 42 + br i1 %exitcond, label %return, label %for.body + +for.body: + %inc = add i32 %iv, 1 + br label %for.cond + +return: + ret void +} + !12 = !{i32 1} -- GitLab From 9b2a349991a87b2d9d576b0b1f63f357870449b1 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Wed, 5 Jun 2024 12:47:24 +0200 Subject: [PATCH 008/675] [MLIR][ModuleTranslation] Add disableVerification parameter (NFC) (#94445) This commit adds a boolean parameter that allows downstream users to disable the verification when translating an MLIR module to LLVM IR. This is helpful for debugging broken LLVM IR modules post translation. --- mlir/include/mlir/Target/LLVMIR/Export.h | 3 ++- mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h | 3 ++- mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/mlir/include/mlir/Target/LLVMIR/Export.h b/mlir/include/mlir/Target/LLVMIR/Export.h index f9a6db0fc94d..224496865513 100644 --- a/mlir/include/mlir/Target/LLVMIR/Export.h +++ b/mlir/include/mlir/Target/LLVMIR/Export.h @@ -26,7 +26,8 @@ class Operation; /// LLVMTranslationDialectInterface. std::unique_ptr translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext, - llvm::StringRef name = "LLVMDialectModule"); + llvm::StringRef name = "LLVMDialectModule", + bool disableVerification = false); } // namespace mlir #endif // MLIR_TARGET_LLVMIR_EXPORT_H diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h index 310a43e0de96..85fdfed3bdbe 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h @@ -57,7 +57,8 @@ class ComdatSelectorOp; /// needs to look up block and function mappings. class ModuleTranslation { friend std::unique_ptr - mlir::translateModuleToLLVMIR(Operation *, llvm::LLVMContext &, StringRef); + mlir::translateModuleToLLVMIR(Operation *, llvm::LLVMContext &, StringRef, + bool); public: /// Stores the mapping between a function name and its LLVM IR representation. diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index 176821f82434..7b86b250c294 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -1818,7 +1818,7 @@ prepareLLVMModule(Operation *m, llvm::LLVMContext &llvmContext, std::unique_ptr mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext, - StringRef name) { + StringRef name, bool disableVerification) { if (!satisfiesLLVMModule(module)) { module->emitOpError("can not be translated to an LLVMIR module"); return nullptr; @@ -1867,7 +1867,8 @@ mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext, if (failed(translator.convertFunctions())) return nullptr; - if (llvm::verifyModule(*translator.llvmModule, &llvm::errs())) + if (!disableVerification && + llvm::verifyModule(*translator.llvmModule, &llvm::errs())) return nullptr; return std::move(translator.llvmModule); -- GitLab From 54b20cbb95fec00ebc0cc83c8d7ca885294c1016 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 5 Jun 2024 11:57:55 +0100 Subject: [PATCH 009/675] [DAG] computeKnownBits - abds(x, y) will be zero in the upper bits if x and y are sign-extended (#94448) As reported on #94442 - if x and y have more than one signbit, then the upper bits of its absolute value are guaranteed to be zero Sibling PR to #94382 Alive2: https://alive2.llvm.org/ce/z/7_z2Vc Fixes #94442 --- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 7 ++++ llvm/test/CodeGen/AArch64/neon-abd.ll | 34 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 6c9b64810c33..4a6a431696b5 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -3477,6 +3477,13 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known = KnownBits::abds(Known, Known2); + unsigned SignBits1 = + ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); + if (SignBits1 == 1) + break; + unsigned SignBits0 = + ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); + Known.Zero.setHighBits(std::min(SignBits0, SignBits1) - 1); break; } case ISD::UMUL_LOHI: { diff --git a/llvm/test/CodeGen/AArch64/neon-abd.ll b/llvm/test/CodeGen/AArch64/neon-abd.ll index 901cb8adc23f..f743bae84053 100644 --- a/llvm/test/CodeGen/AArch64/neon-abd.ll +++ b/llvm/test/CodeGen/AArch64/neon-abd.ll @@ -554,6 +554,40 @@ define <16 x i8> @umaxmin_v16i8_com1(<16 x i8> %0, <16 x i8> %1) { ret <16 x i8> %sub } +; (abds x, y) upper bits are known zero if x and y have extra sign bits +define <4 x i16> @combine_sabd_4h_zerosign(<4 x i16> %a, <4 x i16> %b) #0 { +; CHECK-LABEL: combine_sabd_4h_zerosign: +; CHECK: // %bb.0: +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: ret + %a.ext = ashr <4 x i16> %a, + %b.ext = ashr <4 x i16> %b, + %max = tail call <4 x i16> @llvm.smax.v4i16(<4 x i16> %a.ext, <4 x i16> %b.ext) + %min = tail call <4 x i16> @llvm.smin.v4i16(<4 x i16> %a.ext, <4 x i16> %b.ext) + %sub = sub <4 x i16> %max, %min + %mask = and <4 x i16> %sub, + ret <4 x i16> %mask +} + +; negative test - mask extends beyond known zero bits +define <2 x i32> @combine_sabd_2s_zerosign_negative(<2 x i32> %a, <2 x i32> %b) { +; CHECK-LABEL: combine_sabd_2s_zerosign_negative: +; CHECK: // %bb.0: +; CHECK-NEXT: sshr v0.2s, v0.2s, #3 +; CHECK-NEXT: sshr v1.2s, v1.2s, #15 +; CHECK-NEXT: mvni v2.2s, #7, msl #16 +; CHECK-NEXT: sabd v0.2s, v0.2s, v1.2s +; CHECK-NEXT: and v0.8b, v0.8b, v2.8b +; CHECK-NEXT: ret + %a.ext = ashr <2 x i32> %a, + %b.ext = ashr <2 x i32> %b, + %max = tail call <2 x i32> @llvm.smax.v2i32(<2 x i32> %a.ext, <2 x i32> %b.ext) + %min = tail call <2 x i32> @llvm.smin.v2i32(<2 x i32> %a.ext, <2 x i32> %b.ext) + %sub = sub <2 x i32> %max, %min + %mask = and <2 x i32> %sub, ; 0xFFF80000 + ret <2 x i32> %mask +} + declare <8 x i8> @llvm.abs.v8i8(<8 x i8>, i1) declare <16 x i8> @llvm.abs.v16i8(<16 x i8>, i1) -- GitLab From 1ea568895aa106a61e84607edfd52c3ebf4b59bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Wed, 5 Jun 2024 13:01:35 +0200 Subject: [PATCH 010/675] [clang][Interp][NFC] Mark failed globals as uninitialized This happens automatically if anything _before_ the Ret op fails, but in this case we have to un-initialize it again. --- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index 3eb7e7544df7..3671c41ae703 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -3080,12 +3080,22 @@ bool ByteCodeExprGen::visitDecl(const VarDecl *VD) { } } - // Return the value - if (VarT) - return this->emitRet(*VarT, VD); - - // Return non-primitive values as pointers here. - return this->emitRet(PT_Ptr, VD); + // Return the value. + if (!this->emitRet(VarT.value_or(PT_Ptr), VD)) { + // If the Ret above failed and this is a global variable, mark it as + // uninitialized, even everything else succeeded. + if (Context::shouldBeGloballyIndexed(VD)) { + auto GlobalIndex = P.getGlobal(VD); + assert(GlobalIndex); + Block *GlobalBlock = P.getGlobal(*GlobalIndex); + InlineDescriptor &ID = + *reinterpret_cast(GlobalBlock->rawData()); + ID.IsInitialized = false; + GlobalBlock->invokeDtor(); + } + return false; + } + return true; } template -- GitLab From d4d3239d982e15e039d3958b4202b13203df26bd Mon Sep 17 00:00:00 2001 From: paperchalice Date: Wed, 5 Jun 2024 19:24:19 +0800 Subject: [PATCH 011/675] [NewPM][CodeGen] Port `localstackalloc` to new pass manager (#94303) There are two AArch64 tests use `-start-before` and `-print-after`. Rest tests uses `--passes` to test this pass. --- .../llvm/CodeGen/LocalStackSlotAllocation.h | 23 +++++++++++++ llvm/include/llvm/Passes/CodeGenPassBuilder.h | 5 +-- .../llvm/Passes/MachinePassRegistry.def | 2 +- llvm/lib/CodeGen/LocalStackSlotAllocation.cpp | 33 +++++++++++++++---- llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/test/CodeGen/AArch64/aarch64st1.mir | 1 + .../CodeGen/AArch64/sve-localstackalloc.mir | 1 + 7 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 llvm/include/llvm/CodeGen/LocalStackSlotAllocation.h diff --git a/llvm/include/llvm/CodeGen/LocalStackSlotAllocation.h b/llvm/include/llvm/CodeGen/LocalStackSlotAllocation.h new file mode 100644 index 000000000000..bf5225d3e99a --- /dev/null +++ b/llvm/include/llvm/CodeGen/LocalStackSlotAllocation.h @@ -0,0 +1,23 @@ +//===- LocalStackSlotAllocation.h -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_LOCALSTACKSLOTALLOCATION_H +#define LLVM_CODEGEN_LOCALSTACKSLOTALLOCATION_H + +#include "llvm/CodeGen/MachinePassManager.h" + +namespace llvm { + +class LocalStackSlotAllocationPass + : public PassInfoMixin { +public: + PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &); +}; + +} // namespace llvm +#endif // LLVM_CODEGEN_LOCALSTACKSLOTALLOCATION_H diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index 3c4723a0513c..7542986837ec 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -36,6 +36,7 @@ #include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/InterleavedLoadCombine.h" #include "llvm/CodeGen/JMCInstrumenter.h" +#include "llvm/CodeGen/LocalStackSlotAllocation.h" #include "llvm/CodeGen/LowerEmuTLS.h" #include "llvm/CodeGen/MIRPrinter.h" #include "llvm/CodeGen/MachineFunctionAnalysis.h" @@ -881,7 +882,7 @@ Error CodeGenPassBuilder::addMachinePasses( } else { // If the target requests it, assign local variables to stack slots relative // to one another and simplify frame index references where possible. - addPass(LocalStackSlotPass()); + addPass(LocalStackSlotAllocationPass()); } if (TM.Options.EnableIPRA) @@ -995,7 +996,7 @@ void CodeGenPassBuilder::addMachineSSAOptimization( // If the target requests it, assign local variables to stack slots relative // to one another and simplify frame index references where possible. - addPass(LocalStackSlotPass()); + addPass(LocalStackSlotAllocationPass()); // With optimization, dead code should already be eliminated. However // there is one known exception: lowered code for arguments that are only diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index fc2beb728645..4152c35ded6c 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -125,6 +125,7 @@ MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PI #endif MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass()) MACHINE_FUNCTION_PASS("finalize-isel", FinalizeISelPass()) +MACHINE_FUNCTION_PASS("localstackalloc", LocalStackSlotAllocationPass()) MACHINE_FUNCTION_PASS("no-op-machine-function", NoOpMachineFunctionPass()) MACHINE_FUNCTION_PASS("print", PrintMIRPass()) MACHINE_FUNCTION_PASS("require-all-machine-function-properties", @@ -182,7 +183,6 @@ DUMMY_MACHINE_FUNCTION_PASS("kcfi", MachineKCFIPass) DUMMY_MACHINE_FUNCTION_PASS("legalizer", LegalizerPass) DUMMY_MACHINE_FUNCTION_PASS("livedebugvalues", LiveDebugValuesPass) DUMMY_MACHINE_FUNCTION_PASS("liveintervals", LiveIntervalsPass) -DUMMY_MACHINE_FUNCTION_PASS("localstackalloc", LocalStackSlotPass) DUMMY_MACHINE_FUNCTION_PASS("lrshrink", LiveRangeShrinkPass) DUMMY_MACHINE_FUNCTION_PASS("machine-combiner", MachineCombinerPass) DUMMY_MACHINE_FUNCTION_PASS("machine-cp", MachineCopyPropagationPass) diff --git a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp index e491ed12034d..0bb7953efd52 100644 --- a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp +++ b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp @@ -13,6 +13,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CodeGen/LocalStackSlotAllocation.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" @@ -71,7 +72,7 @@ namespace { int getFrameIndex() const { return FrameIdx; } }; - class LocalStackSlotPass: public MachineFunctionPass { + class LocalStackSlotImpl { SmallVector LocalOffsets; /// StackObjSet - A set of stack object indexes @@ -86,6 +87,11 @@ namespace { void calculateFrameObjectOffsets(MachineFunction &Fn); bool insertFrameReferenceRegisters(MachineFunction &Fn); + public: + bool runOnMachineFunction(MachineFunction &MF); + }; + + class LocalStackSlotPass : public MachineFunctionPass { public: static char ID; // Pass identification, replacement for typeid @@ -93,7 +99,9 @@ namespace { initializeLocalStackSlotPassPass(*PassRegistry::getPassRegistry()); } - bool runOnMachineFunction(MachineFunction &MF) override; + bool runOnMachineFunction(MachineFunction &MF) override { + return LocalStackSlotImpl().runOnMachineFunction(MF); + } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); @@ -103,13 +111,24 @@ namespace { } // end anonymous namespace +PreservedAnalyses +LocalStackSlotAllocationPass::run(MachineFunction &MF, + MachineFunctionAnalysisManager &) { + bool Changed = LocalStackSlotImpl().runOnMachineFunction(MF); + if (!Changed) + return PreservedAnalyses::all(); + auto PA = getMachineFunctionPassPreservedAnalyses(); + PA.preserveSet(); + return PA; +} + char LocalStackSlotPass::ID = 0; char &llvm::LocalStackSlotAllocationID = LocalStackSlotPass::ID; INITIALIZE_PASS(LocalStackSlotPass, DEBUG_TYPE, "Local Stack Slot Allocation", false, false) -bool LocalStackSlotPass::runOnMachineFunction(MachineFunction &MF) { +bool LocalStackSlotImpl::runOnMachineFunction(MachineFunction &MF) { MachineFrameInfo &MFI = MF.getFrameInfo(); const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); unsigned LocalObjectCount = MFI.getObjectIndexEnd(); @@ -139,7 +158,7 @@ bool LocalStackSlotPass::runOnMachineFunction(MachineFunction &MF) { } /// AdjustStackOffset - Helper function used to adjust the stack frame offset. -void LocalStackSlotPass::AdjustStackOffset(MachineFrameInfo &MFI, int FrameIdx, +void LocalStackSlotImpl::AdjustStackOffset(MachineFrameInfo &MFI, int FrameIdx, int64_t &Offset, bool StackGrowsDown, Align &MaxAlign) { // If the stack grows down, add the object size to find the lowest address. @@ -171,7 +190,7 @@ void LocalStackSlotPass::AdjustStackOffset(MachineFrameInfo &MFI, int FrameIdx, /// AssignProtectedObjSet - Helper function to assign large stack objects (i.e., /// those required to be close to the Stack Protector) to stack offsets. -void LocalStackSlotPass::AssignProtectedObjSet( +void LocalStackSlotImpl::AssignProtectedObjSet( const StackObjSet &UnassignedObjs, SmallSet &ProtectedObjs, MachineFrameInfo &MFI, bool StackGrowsDown, int64_t &Offset, Align &MaxAlign) { @@ -183,7 +202,7 @@ void LocalStackSlotPass::AssignProtectedObjSet( /// calculateFrameObjectOffsets - Calculate actual frame offsets for all of the /// abstract stack objects. -void LocalStackSlotPass::calculateFrameObjectOffsets(MachineFunction &Fn) { +void LocalStackSlotImpl::calculateFrameObjectOffsets(MachineFunction &Fn) { // Loop over all of the stack objects, assigning sequential addresses... MachineFrameInfo &MFI = Fn.getFrameInfo(); const TargetFrameLowering &TFI = *Fn.getSubtarget().getFrameLowering(); @@ -281,7 +300,7 @@ lookupCandidateBaseReg(unsigned BaseReg, return TRI->isFrameOffsetLegal(&MI, BaseReg, Offset); } -bool LocalStackSlotPass::insertFrameReferenceRegisters(MachineFunction &Fn) { +bool LocalStackSlotImpl::insertFrameReferenceRegisters(MachineFunction &Fn) { // Scan the function's instructions looking for frame index references. // For each, ask the target if it wants a virtual base register for it // based on what we can tell it about where the local will end up in the diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 09231504ef90..316d05bf1dc3 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -90,6 +90,7 @@ #include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/InterleavedLoadCombine.h" #include "llvm/CodeGen/JMCInstrumenter.h" +#include "llvm/CodeGen/LocalStackSlotAllocation.h" #include "llvm/CodeGen/LowerEmuTLS.h" #include "llvm/CodeGen/MIRPrinter.h" #include "llvm/CodeGen/MachineFunctionAnalysis.h" diff --git a/llvm/test/CodeGen/AArch64/aarch64st1.mir b/llvm/test/CodeGen/AArch64/aarch64st1.mir index 9482a42e69f6..22a024d37bc6 100644 --- a/llvm/test/CodeGen/AArch64/aarch64st1.mir +++ b/llvm/test/CodeGen/AArch64/aarch64st1.mir @@ -1,5 +1,6 @@ # Check that it doesn't crash with unhandled opcode error, see pr52249 # RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass localstackalloc -o - %s | FileCheck %s +# RUN: llc -mtriple=aarch64-none-linux-gnu -passes=localstackalloc -o - %s | FileCheck %s --- | define void @test_st1_to_sp(<2 x i32> %a, <4 x i16> %b, <8 x i8> %c, <2 x i64> %d) gc "statepoint-example" { entry: ret void } diff --git a/llvm/test/CodeGen/AArch64/sve-localstackalloc.mir b/llvm/test/CodeGen/AArch64/sve-localstackalloc.mir index 6063c8dfc792..b4105bd62f8b 100644 --- a/llvm/test/CodeGen/AArch64/sve-localstackalloc.mir +++ b/llvm/test/CodeGen/AArch64/sve-localstackalloc.mir @@ -1,4 +1,5 @@ # RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -run-pass=localstackalloc -o - %s | FileCheck %s +# RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -passes=localstackalloc -o - %s | FileCheck %s --- | ; ModuleID = '' -- GitLab From 5f2aa912766e4d48a5b46afa7ad9e99a41a51ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Wed, 5 Jun 2024 13:14:18 +0200 Subject: [PATCH 012/675] [clang][Interp][NFC] Don't try to dump uninitialized global variables They don't contain anything useful. --- clang/lib/AST/Interp/Disasm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/AST/Interp/Disasm.cpp b/clang/lib/AST/Interp/Disasm.cpp index ccdc96a79436..e442c6c709f1 100644 --- a/clang/lib/AST/Interp/Disasm.cpp +++ b/clang/lib/AST/Interp/Disasm.cpp @@ -150,7 +150,7 @@ LLVM_DUMP_METHOD void Program::dump(llvm::raw_ostream &OS) const { } Desc->dump(OS); OS << "\n"; - if (Desc->isPrimitive() && !Desc->isDummy()) { + if (GP.isInitialized() && Desc->isPrimitive() && !Desc->isDummy()) { OS << " "; { ColorScope SC(OS, true, {llvm::raw_ostream::BRIGHT_CYAN, false}); -- GitLab From 145815c180fc82c5a55bf568d01d98d250490a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Wed, 5 Jun 2024 13:21:40 +0200 Subject: [PATCH 013/675] [clang][Interp][NFC] Move EvaluationResult::dump() to Disasm.cpp Where all the other dump() functions live. --- clang/lib/AST/Interp/Disasm.cpp | 42 +++++++++++++++++++++++ clang/lib/AST/Interp/EvaluationResult.cpp | 41 ---------------------- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/clang/lib/AST/Interp/Disasm.cpp b/clang/lib/AST/Interp/Disasm.cpp index e442c6c709f1..3f8a92ed2f0b 100644 --- a/clang/lib/AST/Interp/Disasm.cpp +++ b/clang/lib/AST/Interp/Disasm.cpp @@ -11,6 +11,8 @@ //===----------------------------------------------------------------------===// #include "Boolean.h" +#include "Context.h" +#include "EvaluationResult.h" #include "Floating.h" #include "Function.h" #include "FunctionPointer.h" @@ -305,3 +307,43 @@ LLVM_DUMP_METHOD void Block::dump(llvm::raw_ostream &OS) const { OS << " Extern: " << IsExtern << "\n"; OS << " Initialized: " << IsInitialized << "\n"; } + +LLVM_DUMP_METHOD void EvaluationResult::dump() const { + assert(Ctx); + auto &OS = llvm::errs(); + const ASTContext &ASTCtx = Ctx->getASTContext(); + + switch (Kind) { + case Empty: + OS << "Empty\n"; + break; + case RValue: + OS << "RValue: "; + std::get(Value).dump(OS, ASTCtx); + break; + case LValue: { + assert(Source); + QualType SourceType; + if (const auto *D = Source.dyn_cast()) { + if (const auto *VD = dyn_cast(D)) + SourceType = VD->getType(); + } else if (const auto *E = Source.dyn_cast()) { + SourceType = E->getType(); + } + + OS << "LValue: "; + if (const auto *P = std::get_if(&Value)) + P->toAPValue().printPretty(OS, ASTCtx, SourceType); + else if (const auto *FP = std::get_if(&Value)) // Nope + FP->toAPValue().printPretty(OS, ASTCtx, SourceType); + OS << "\n"; + break; + } + case Invalid: + OS << "Invalid\n"; + break; + case Valid: + OS << "Valid\n"; + break; + } +} diff --git a/clang/lib/AST/Interp/EvaluationResult.cpp b/clang/lib/AST/Interp/EvaluationResult.cpp index 150a793da881..c04b0fc0a112 100644 --- a/clang/lib/AST/Interp/EvaluationResult.cpp +++ b/clang/lib/AST/Interp/EvaluationResult.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "EvaluationResult.h" -#include "Context.h" #include "InterpState.h" #include "Record.h" #include "clang/AST/ExprCXX.h" @@ -159,45 +158,5 @@ bool EvaluationResult::checkFullyInitialized(InterpState &S, return CheckArrayInitialized(S, InitLoc, Ptr, CAT); } -void EvaluationResult::dump() const { - assert(Ctx); - auto &OS = llvm::errs(); - const ASTContext &ASTCtx = Ctx->getASTContext(); - - switch (Kind) { - case Empty: - OS << "Empty\n"; - break; - case RValue: - OS << "RValue: "; - std::get(Value).dump(OS, ASTCtx); - break; - case LValue: { - assert(Source); - QualType SourceType; - if (const auto *D = Source.dyn_cast()) { - if (const auto *VD = dyn_cast(D)) - SourceType = VD->getType(); - } else if (const auto *E = Source.dyn_cast()) { - SourceType = E->getType(); - } - - OS << "LValue: "; - if (const auto *P = std::get_if(&Value)) - P->toAPValue().printPretty(OS, ASTCtx, SourceType); - else if (const auto *FP = std::get_if(&Value)) // Nope - FP->toAPValue().printPretty(OS, ASTCtx, SourceType); - OS << "\n"; - break; - } - case Invalid: - OS << "Invalid\n"; - break; - case Valid: - OS << "Valid\n"; - break; - } -} - } // namespace interp } // namespace clang -- GitLab From a44d7406f45fd3e5af45de116aed03b0bf7a881f Mon Sep 17 00:00:00 2001 From: Carl Ritson Date: Wed, 5 Jun 2024 19:47:04 +0900 Subject: [PATCH 014/675] [AMDGPU][NFC] Pre-commit test for PR #94133 --- llvm/test/CodeGen/AMDGPU/wqm.ll | 91 +++++++++++++++++++++++++++++++-- 1 file changed, 87 insertions(+), 4 deletions(-) diff --git a/llvm/test/CodeGen/AMDGPU/wqm.ll b/llvm/test/CodeGen/AMDGPU/wqm.ll index 95dfb12c8dba..6fcf5067b022 100644 --- a/llvm/test/CodeGen/AMDGPU/wqm.ll +++ b/llvm/test/CodeGen/AMDGPU/wqm.ll @@ -2936,6 +2936,89 @@ ENDIF: ret float %r } +; WQM -> StrictWQM transition must be preserved because kill breaks WQM mask +define amdgpu_ps float @test_strict_wqm_within_wqm_with_kill(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data, i32 %wqm_data) { +; GFX9-W64-LABEL: test_strict_wqm_within_wqm_with_kill: +; GFX9-W64: ; %bb.0: ; %main_body +; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec +; GFX9-W64-NEXT: s_mov_b64 s[14:15], exec +; GFX9-W64-NEXT: s_wqm_b64 exec, exec +; GFX9-W64-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-W64-NEXT: s_mov_b64 exec, s[14:15] +; GFX9-W64-NEXT: s_wqm_b64 exec, exec +; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 +; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 +; GFX9-W64-NEXT: s_waitcnt vmcnt(0) +; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 +; GFX9-W64-NEXT: s_xor_b64 s[0:1], vcc, exec +; GFX9-W64-NEXT: s_andn2_b64 s[12:13], s[12:13], s[0:1] +; GFX9-W64-NEXT: s_cbranch_scc0 .LBB51_2 +; GFX9-W64-NEXT: ; %bb.1: ; %main_body +; GFX9-W64-NEXT: s_and_b64 exec, exec, vcc +; GFX9-W64-NEXT: ds_swizzle_b32 v3, v3 offset:swizzle(SWAP,2) +; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-W64-NEXT: v_mov_b32_e32 v1, v3 +; GFX9-W64-NEXT: v_cvt_f32_i32_e32 v1, v1 +; GFX9-W64-NEXT: s_waitcnt vmcnt(0) +; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec +; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] +; GFX9-W64-NEXT: s_branch .LBB51_3 +; GFX9-W64-NEXT: .LBB51_2: +; GFX9-W64-NEXT: s_mov_b64 exec, 0 +; GFX9-W64-NEXT: exp null off, off, off, off done vm +; GFX9-W64-NEXT: s_endpgm +; GFX9-W64-NEXT: .LBB51_3: +; +; GFX10-W32-LABEL: test_strict_wqm_within_wqm_with_kill: +; GFX10-W32: ; %bb.0: ; %main_body +; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo +; GFX10-W32-NEXT: s_mov_b32 s13, exec_lo +; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo +; GFX10-W32-NEXT: v_mov_b32_e32 v3, v2 +; GFX10-W32-NEXT: s_mov_b32 exec_lo, s13 +; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo +; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D +; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 +; GFX10-W32-NEXT: s_waitcnt vmcnt(0) +; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D +; GFX10-W32-NEXT: s_xor_b32 s0, vcc_lo, exec_lo +; GFX10-W32-NEXT: s_andn2_b32 s12, s12, s0 +; GFX10-W32-NEXT: s_cbranch_scc0 .LBB51_2 +; GFX10-W32-NEXT: ; %bb.1: ; %main_body +; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, vcc_lo +; GFX10-W32-NEXT: ds_swizzle_b32 v3, v3 offset:swizzle(SWAP,2) +; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-W32-NEXT: v_mov_b32_e32 v1, v3 +; GFX10-W32-NEXT: v_cvt_f32_i32_e32 v1, v1 +; GFX10-W32-NEXT: s_waitcnt vmcnt(0) +; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec +; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 +; GFX10-W32-NEXT: s_branch .LBB51_3 +; GFX10-W32-NEXT: .LBB51_2: +; GFX10-W32-NEXT: s_mov_b32 exec_lo, 0 +; GFX10-W32-NEXT: exp null off, off, off, off done vm +; GFX10-W32-NEXT: s_endpgm +; GFX10-W32-NEXT: .LBB51_3: +main_body: + %c.bc = bitcast i32 %c to float + %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 + %tex0 = extractelement <4 x float> %tex, i32 0 + %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 + %cmp = icmp eq i32 %z, 0 + call void @llvm.amdgcn.kill(i1 %cmp) + %dataf = extractelement <4 x float> %dtex, i32 0 + %data2 = call i32 @llvm.amdgcn.ds.swizzle(i32 %wqm_data, i32 2079) + %data3 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %data2) + %data3f = sitofp i32 %data3 to float + %result.f = fadd float %dataf, %data3f + %result.i = bitcast float %result.f to i32 + %result.wqm = call i32 @llvm.amdgcn.wqm.i32(i32 %result.i) + %result = bitcast i32 %result.wqm to float + ret float %result +} + ;TODO: StrictWQM -> WQM transition could be improved. WQM could use the exec from the previous state instead of calling s_wqm again. define amdgpu_ps float @test_strict_wqm_strict_wwm_wqm(i32 inreg %idx0, i32 inreg %idx1, ptr addrspace(8) inreg %res, ptr addrspace(8) inreg %res2, float %inp, <8 x i32> inreg %res3) { ; GFX9-W64-LABEL: test_strict_wqm_strict_wwm_wqm: @@ -3281,9 +3364,9 @@ define amdgpu_ps void @test_for_deactivating_lanes_in_wave32(ptr addrspace(6) in ; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-W64-NEXT: v_cmp_le_f32_e64 vcc, s0, 0 ; GFX9-W64-NEXT: s_andn2_b64 s[4:5], exec, vcc -; GFX9-W64-NEXT: s_cbranch_scc0 .LBB54_1 +; GFX9-W64-NEXT: s_cbranch_scc0 .LBB55_1 ; GFX9-W64-NEXT: s_endpgm -; GFX9-W64-NEXT: .LBB54_1: +; GFX9-W64-NEXT: .LBB55_1: ; GFX9-W64-NEXT: s_mov_b64 exec, 0 ; GFX9-W64-NEXT: exp null off, off, off, off done vm ; GFX9-W64-NEXT: s_endpgm @@ -3297,9 +3380,9 @@ define amdgpu_ps void @test_for_deactivating_lanes_in_wave32(ptr addrspace(6) in ; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-W32-NEXT: v_cmp_le_f32_e64 vcc_lo, s0, 0 ; GFX10-W32-NEXT: s_andn2_b32 s4, exec_lo, vcc_lo -; GFX10-W32-NEXT: s_cbranch_scc0 .LBB54_1 +; GFX10-W32-NEXT: s_cbranch_scc0 .LBB55_1 ; GFX10-W32-NEXT: s_endpgm -; GFX10-W32-NEXT: .LBB54_1: +; GFX10-W32-NEXT: .LBB55_1: ; GFX10-W32-NEXT: s_mov_b32 exec_lo, 0 ; GFX10-W32-NEXT: exp null off, off, off, off done vm ; GFX10-W32-NEXT: s_endpgm -- GitLab From 3388c5aadd8583b5a596576c52be886104d557f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Wed, 5 Jun 2024 13:31:29 +0200 Subject: [PATCH 015/675] [clang][Interp][NFC] Add missing assertion to Block ctor We have this assertion in all the other constructors. --- clang/lib/AST/Interp/InterpBlock.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/clang/lib/AST/Interp/InterpBlock.h b/clang/lib/AST/Interp/InterpBlock.h index 506034e880d0..2bb195648a9a 100644 --- a/clang/lib/AST/Interp/InterpBlock.h +++ b/clang/lib/AST/Interp/InterpBlock.h @@ -125,13 +125,15 @@ public: void dump() const { dump(llvm::errs()); } void dump(llvm::raw_ostream &OS) const; -protected: +private: friend class Pointer; friend class DeadBlock; friend class InterpState; Block(const Descriptor *Desc, bool IsExtern, bool IsStatic, bool IsDead) - : IsStatic(IsStatic), IsExtern(IsExtern), IsDead(true), Desc(Desc) {} + : IsStatic(IsStatic), IsExtern(IsExtern), IsDead(true), Desc(Desc) { + assert(Desc); + } /// Deletes a dead block at the end of its lifetime. void cleanup(); -- GitLab From a16d33eaebb3fdbc9435c125c206372c8a7374d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Wed, 5 Jun 2024 13:36:10 +0200 Subject: [PATCH 016/675] [clang][Interp][NFC] Don't invoke block dtor on uninitialized globals That can't ever work. --- clang/lib/AST/Interp/Program.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clang/lib/AST/Interp/Program.h b/clang/lib/AST/Interp/Program.h index 36b5a1faa513..ec7c0744b885 100644 --- a/clang/lib/AST/Interp/Program.h +++ b/clang/lib/AST/Interp/Program.h @@ -45,7 +45,8 @@ public: // but primitive arrays might have an InitMap* heap allocated and // that needs to be freed. for (Global *G : Globals) - G->block()->invokeDtor(); + if (Block *B = G->block(); B->isInitialized()) + B->invokeDtor(); // Records might actually allocate memory themselves, but they // are allocated using a BumpPtrAllocator. Call their desctructors -- GitLab From c70fa55bed45fc0cc0063e9f0bf93f163b5a1962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Wed, 5 Jun 2024 13:56:23 +0200 Subject: [PATCH 017/675] [clang][Interp][NFC] Add cleanup() infrastructure to EvalEmitter Unused for now. --- clang/lib/AST/Interp/Context.cpp | 3 +++ clang/lib/AST/Interp/EvalEmitter.cpp | 5 +++++ clang/lib/AST/Interp/EvalEmitter.h | 3 +++ clang/lib/AST/Interp/InterpState.cpp | 2 ++ clang/lib/AST/Interp/InterpState.h | 2 ++ 5 files changed, 15 insertions(+) diff --git a/clang/lib/AST/Interp/Context.cpp b/clang/lib/AST/Interp/Context.cpp index 4ecfa0f9bfd7..b0b22b059b77 100644 --- a/clang/lib/AST/Interp/Context.cpp +++ b/clang/lib/AST/Interp/Context.cpp @@ -46,6 +46,7 @@ bool Context::evaluateAsRValue(State &Parent, const Expr *E, APValue &Result) { auto Res = C.interpretExpr(E, /*ConvertResultToRValue=*/E->isGLValue()); if (Res.isInvalid()) { + C.cleanup(); Stk.clear(); return false; } @@ -70,6 +71,7 @@ bool Context::evaluate(State &Parent, const Expr *E, APValue &Result) { auto Res = C.interpretExpr(E); if (Res.isInvalid()) { + C.cleanup(); Stk.clear(); return false; } @@ -97,6 +99,7 @@ bool Context::evaluateAsInitializer(State &Parent, const VarDecl *VD, (VD->getType()->isRecordType() || VD->getType()->isArrayType()); auto Res = C.interpretDecl(VD, CheckGlobalInitialized); if (Res.isInvalid()) { + C.cleanup(); Stk.clear(); return false; } diff --git a/clang/lib/AST/Interp/EvalEmitter.cpp b/clang/lib/AST/Interp/EvalEmitter.cpp index 388c3612f292..6d8aa3f20f01 100644 --- a/clang/lib/AST/Interp/EvalEmitter.cpp +++ b/clang/lib/AST/Interp/EvalEmitter.cpp @@ -32,6 +32,11 @@ EvalEmitter::~EvalEmitter() { } } +/// Clean up all our resources. This needs to done in failed evaluations before +/// we call InterpStack::clear(), because there might be a Pointer on the stack +/// pointing into a Block in the EvalEmitter. +void EvalEmitter::cleanup() { S.cleanup(); } + EvaluationResult EvalEmitter::interpretExpr(const Expr *E, bool ConvertResultToRValue) { S.setEvalLocation(E->getExprLoc()); diff --git a/clang/lib/AST/Interp/EvalEmitter.h b/clang/lib/AST/Interp/EvalEmitter.h index 116f1d6fc134..98d6026bbcce 100644 --- a/clang/lib/AST/Interp/EvalEmitter.h +++ b/clang/lib/AST/Interp/EvalEmitter.h @@ -38,6 +38,9 @@ public: bool ConvertResultToRValue = false); EvaluationResult interpretDecl(const VarDecl *VD, bool CheckFullyInitialized); + /// Clean up all resources. + void cleanup(); + InterpState &getState() { return S; } protected: diff --git a/clang/lib/AST/Interp/InterpState.cpp b/clang/lib/AST/Interp/InterpState.cpp index 2cb87ef07fe5..550bc9f1a84b 100644 --- a/clang/lib/AST/Interp/InterpState.cpp +++ b/clang/lib/AST/Interp/InterpState.cpp @@ -33,6 +33,8 @@ InterpState::~InterpState() { } } +void InterpState::cleanup() {} + Frame *InterpState::getCurrentFrame() { if (Current && Current->Caller) return Current; diff --git a/clang/lib/AST/Interp/InterpState.h b/clang/lib/AST/Interp/InterpState.h index d483c60c58e2..0938a723a76d 100644 --- a/clang/lib/AST/Interp/InterpState.h +++ b/clang/lib/AST/Interp/InterpState.h @@ -39,6 +39,8 @@ public: ~InterpState(); + void cleanup(); + InterpState(const InterpState &) = delete; InterpState &operator=(const InterpState &) = delete; -- GitLab From 6168e82c1e17c58ebc9c0b6c8f2273fd9a610977 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Wed, 5 Jun 2024 14:08:00 +0200 Subject: [PATCH 018/675] [MLIR][LLVM] Add inlining support for loop annotations (#94447) This commit extends the LLVM dialect's inliner interface support updating loop annotation attributes. This is necessary because the loop annotations can contain debug locations, which are verified by LLVM's verifier. LLVM requires these locations to have the same scope as the function this attribute is contained in. --- mlir/lib/Dialect/LLVMIR/IR/LLVMInlining.cpp | 52 +++++++++++++++++ .../LLVMIR/inlining-loop-annotation.mlir | 56 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 mlir/test/Dialect/LLVMIR/inlining-loop-annotation.mlir diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMInlining.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMInlining.cpp index 5552dc5e244b..cf3369d053fa 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMInlining.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMInlining.cpp @@ -511,6 +511,57 @@ static void handleAccessGroups(Operation *call, accessGroupOpInterface.getAccessGroupsOrNull(), accessGroups)); } +/// Updates locations inside loop annotations to reflect that they were inlined. +static void +handleLoopAnnotations(Operation *call, + iterator_range inlinedBlocks) { + // Attempt to extract a DISubprogram from the callee. + auto func = call->getParentOfType(); + if (!func) + return; + LocationAttr funcLoc = func->getLoc(); + auto fusedLoc = dyn_cast_if_present(funcLoc); + if (!fusedLoc) + return; + auto scope = + dyn_cast_if_present(fusedLoc.getMetadata()); + if (!scope) + return; + + // Helper to build a new fused location that reflects the inlining of the loop + // annotation. + auto updateLoc = [&](FusedLoc loc) -> FusedLoc { + if (!loc) + return {}; + Location callSiteLoc = CallSiteLoc::get(loc, call->getLoc()); + return FusedLoc::get(loc.getContext(), callSiteLoc, scope); + }; + + AttrTypeReplacer replacer; + replacer.addReplacement([&](LLVM::LoopAnnotationAttr loopAnnotation) + -> std::pair { + FusedLoc newStartLoc = updateLoc(loopAnnotation.getStartLoc()); + FusedLoc newEndLoc = updateLoc(loopAnnotation.getEndLoc()); + if (!newStartLoc && !newEndLoc) + return {loopAnnotation, WalkResult::advance()}; + auto newLoopAnnotation = LLVM::LoopAnnotationAttr::get( + loopAnnotation.getContext(), loopAnnotation.getDisableNonforced(), + loopAnnotation.getVectorize(), loopAnnotation.getInterleave(), + loopAnnotation.getUnroll(), loopAnnotation.getUnrollAndJam(), + loopAnnotation.getLicm(), loopAnnotation.getDistribute(), + loopAnnotation.getPipeline(), loopAnnotation.getPeeled(), + loopAnnotation.getUnswitch(), loopAnnotation.getMustProgress(), + loopAnnotation.getIsVectorized(), newStartLoc, newEndLoc, + loopAnnotation.getParallelAccesses()); + // Needs to advance, as loop annotations can be nested. + return {newLoopAnnotation, WalkResult::advance()}; + }); + + for (Block &block : inlinedBlocks) + for (Operation &op : block) + replacer.recursivelyReplaceElementsIn(&op); +} + /// If `requestedAlignment` is higher than the alignment specified on `alloca`, /// realigns `alloca` if this does not exceed the natural stack alignment. /// Returns the post-alignment of `alloca`, whether it was realigned or not. @@ -784,6 +835,7 @@ struct LLVMInlinerInterface : public DialectInlinerInterface { handleInlinedAllocas(call, inlinedBlocks); handleAliasScopes(call, inlinedBlocks); handleAccessGroups(call, inlinedBlocks); + handleLoopAnnotations(call, inlinedBlocks); } // Keeping this (immutable) state on the interface allows us to look up diff --git a/mlir/test/Dialect/LLVMIR/inlining-loop-annotation.mlir b/mlir/test/Dialect/LLVMIR/inlining-loop-annotation.mlir new file mode 100644 index 000000000000..e218c151400c --- /dev/null +++ b/mlir/test/Dialect/LLVMIR/inlining-loop-annotation.mlir @@ -0,0 +1,56 @@ +// RUN: mlir-opt %s -inline -split-input-file | FileCheck %s + +#di_file = #llvm.di_file<"file.mlir" in "/"> + +// CHECK: #[[START_ORIGINAL:.*]] = loc({{.*}}:42 +#loc1 = loc("test.mlir":42:4) +// CHECK: #[[END_ORIGINAL:.*]] = loc({{.*}}:52 +#loc2 = loc("test.mlir":52:4) +#loc3 = loc("test.mlir":62:4) +// CHECK: #[[CALL_ORIGINAL:.*]] = loc({{.*}}:72 +#loc4 = loc("test.mlir":72:4) + +#di_compile_unit = #llvm.di_compile_unit, sourceLanguage = DW_LANG_C, file = #di_file, isOptimized = false, emissionKind = None> +// CHECK: #[[CALLEE_DI:.*]] = #llvm.di_subprogram<{{.*}}, name = "callee" +#di_subprogram_callee = #llvm.di_subprogram + +// CHECK: #[[CALLER_DI:.*]] = #llvm.di_subprogram<{{.*}}, name = "caller" +#di_subprogram_caller = #llvm.di_subprogram + +// CHECK: #[[START_FUSED_ORIGINAL:.*]] = loc(fused<#[[CALLEE_DI]]>[#[[START_ORIGINAL]] +#start_loc_fused = loc(fused<#di_subprogram_callee>[#loc1]) +// CHECK: #[[END_FUSED_ORIGINAL:.*]] = loc(fused<#[[CALLEE_DI]]>[#[[END_ORIGINAL]] +#end_loc_fused= loc(fused<#di_subprogram_callee>[#loc2]) +#caller_loc= loc(fused<#di_subprogram_caller>[#loc3]) +// CHECK: #[[CALL_FUSED:.*]] = loc(fused<#[[CALLER_DI]]>[#[[CALL_ORIGINAL]] +#call_loc= loc(fused<#di_subprogram_caller>[#loc4]) + +#loopMD = #llvm.loop_annotation< + startLoc = #start_loc_fused, + endLoc = #end_loc_fused> + +// CHECK: #[[START_CALLSITE_LOC:.*]] = loc(callsite(#[[START_FUSED_ORIGINAL]] at #[[CALL_FUSED]] +// CHECK: #[[END_CALLSITE_LOC:.*]] = loc(callsite(#[[END_FUSED_ORIGINAL]] at #[[CALL_FUSED]] +// CHECK: #[[START_FUSED_LOC:.*]] = loc(fused<#[[CALLER_DI]]>[#[[START_CALLSITE_LOC]] +// CHECK: #[[END_FUSED_LOC:.*]] = loc(fused<#[[CALLER_DI]]>[ +// CHECK: #[[LOOP_ANNOT:.*]] = #llvm.loop_annotation< +// CHECK-SAME: startLoc = #[[START_FUSED_LOC]], endLoc = #[[END_FUSED_LOC]]> + +llvm.func @cond() -> i1 + +llvm.func @callee() { + llvm.br ^head +^head: + %c = llvm.call @cond() : () -> i1 + llvm.cond_br %c, ^head, ^exit {loop_annotation = #loopMD} +^exit: + llvm.return +} + +// CHECK: @loop_annotation +llvm.func @loop_annotation() { + // CHECK: llvm.cond_br + // CHECK-SAME: {loop_annotation = #[[LOOP_ANNOT]] + llvm.call @callee() : () -> () loc(#call_loc) + llvm.return +} loc(#caller_loc) -- GitLab From 3614beede1d22cc7d2492a9742d68b210cb75dd1 Mon Sep 17 00:00:00 2001 From: OverMighty Date: Wed, 5 Jun 2024 14:24:23 +0200 Subject: [PATCH 019/675] [libc][math][c23] Add canonicalizef16 C23 math function (#94341) #93566 --- libc/config/linux/aarch64/entrypoints.txt | 1 + libc/config/linux/x86_64/entrypoints.txt | 1 + libc/docs/c23.rst | 2 +- libc/docs/math/index.rst | 2 +- libc/spec/stdc.td | 1 + libc/src/math/CMakeLists.txt | 3 ++- libc/src/math/canonicalizef16.h | 20 +++++++++++++++++++ libc/src/math/generic/CMakeLists.txt | 14 +++++++++++++ libc/src/math/generic/canonicalizef16.cpp | 19 ++++++++++++++++++ libc/test/src/math/smoke/CMakeLists.txt | 15 ++++++++++++++ libc/test/src/math/smoke/CanonicalizeTest.h | 1 + .../src/math/smoke/canonicalizef16_test.cpp | 13 ++++++++++++ 12 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 libc/src/math/canonicalizef16.h create mode 100644 libc/src/math/generic/canonicalizef16.cpp create mode 100644 libc/test/src/math/smoke/canonicalizef16_test.cpp diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index 74a355407b4b..bf0610f442c5 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -499,6 +499,7 @@ set(TARGET_LIBM_ENTRYPOINTS if(LIBC_TYPES_HAS_FLOAT16) list(APPEND TARGET_LIBM_ENTRYPOINTS # math.h C23 _Float16 entrypoints + libc.src.math.canonicalizef16 libc.src.math.ceilf16 libc.src.math.fabsf16 libc.src.math.floorf16 diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 9a87f3d6d5f0..08720846fbb8 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -532,6 +532,7 @@ set(TARGET_LIBM_ENTRYPOINTS if(LIBC_TYPES_HAS_FLOAT16) list(APPEND TARGET_LIBM_ENTRYPOINTS # math.h C23 _Float16 entrypoints + libc.src.math.canonicalizef16 libc.src.math.ceilf16 libc.src.math.fabsf16 libc.src.math.floorf16 diff --git a/libc/docs/c23.rst b/libc/docs/c23.rst index 33896b5d2fd4..5bbb056ec5c7 100644 --- a/libc/docs/c23.rst +++ b/libc/docs/c23.rst @@ -61,7 +61,7 @@ Additions: * ufromfpx* |check| * nextup* * nextdown* - * canonicalize* + * canonicalize* |check| * fmaximum* * fminimum* * fmaximum_mag* diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst index 1c593dc59180..9040e240c772 100644 --- a/libc/docs/math/index.rst +++ b/libc/docs/math/index.rst @@ -110,7 +110,7 @@ Basic Operations +==================+==================+=================+========================+======================+========================+========================+============================+ | ceil | |check| | |check| | |check| | |check| | |check| | 7.12.9.1 | F.10.6.1 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| canonicalize | |check| | |check| | |check| | | |check| | 7.12.11.7 | F.10.8.7 | +| canonicalize | |check| | |check| | |check| | |check| | |check| | 7.12.11.7 | F.10.8.7 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ | copysign | |check| | |check| | |check| | | |check| | 7.12.11.1 | F.10.8.1 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 91626fa4e04b..df44ed759020 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -679,6 +679,7 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"canonicalize", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"canonicalizef", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"canonicalizel", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"canonicalizef16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"canonicalizef128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, ] >; diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt index e2197544c0de..56174cb81b12 100644 --- a/libc/src/math/CMakeLists.txt +++ b/libc/src/math/CMakeLists.txt @@ -61,8 +61,9 @@ add_math_entrypoint_object(atanhf) add_math_entrypoint_object(canonicalize) add_math_entrypoint_object(canonicalizef) -add_math_entrypoint_object(canonicalizef128) add_math_entrypoint_object(canonicalizel) +add_math_entrypoint_object(canonicalizef16) +add_math_entrypoint_object(canonicalizef128) add_math_entrypoint_object(ceil) add_math_entrypoint_object(ceilf) diff --git a/libc/src/math/canonicalizef16.h b/libc/src/math/canonicalizef16.h new file mode 100644 index 000000000000..102af01c0c2b --- /dev/null +++ b/libc/src/math/canonicalizef16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for canonicalizef16 ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_CANONICALIZEF16_H +#define LLVM_LIBC_SRC_MATH_CANONICALIZEF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +int canonicalizef16(float16 *cx, const float16 *x); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_CANONICALIZEF16_H diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt index 829d5ddc42fe..897ce07baeae 100644 --- a/libc/src/math/generic/CMakeLists.txt +++ b/libc/src/math/generic/CMakeLists.txt @@ -22,6 +22,19 @@ add_entrypoint_object( libc.src.__support.FPUtil.basic_operations ) +add_entrypoint_object( + canonicalizef16 + SRCS + canonicalizef16.cpp + HDRS + ../canonicalizef16.h + COMPILE_OPTIONS + -O3 + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.basic_operations +) + add_entrypoint_object( canonicalizef128 SRCS @@ -31,6 +44,7 @@ add_entrypoint_object( COMPILE_OPTIONS -O3 DEPENDS + libc.src.__support.macros.properties.types libc.src.__support.FPUtil.basic_operations ) diff --git a/libc/src/math/generic/canonicalizef16.cpp b/libc/src/math/generic/canonicalizef16.cpp new file mode 100644 index 000000000000..232e84f7dd68 --- /dev/null +++ b/libc/src/math/generic/canonicalizef16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of canonicalizef16 function ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/canonicalizef16.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, canonicalizef16, (float16 * cx, const float16 *x)) { + return fputil::canonicalize(*cx, *x); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt index 28255d59f39c..b6a3f0b408cc 100644 --- a/libc/test/src/math/smoke/CMakeLists.txt +++ b/libc/test/src/math/smoke/CMakeLists.txt @@ -211,6 +211,21 @@ add_fp_unittest( libc.src.__support.integer_literals ) +add_fp_unittest( + canonicalizef16_test + SUITE + libc-math-smoke-tests + SRCS + canonicalizef16_test.cpp + HDRS + CanonicalizeTest.h + DEPENDS + libc.src.math.canonicalizef16 + libc.src.__support.FPUtil.fp_bits + libc.src.__support.FPUtil.fenv_impl + libc.src.__support.integer_literals +) + add_fp_unittest( canonicalizef128_test SUITE diff --git a/libc/test/src/math/smoke/CanonicalizeTest.h b/libc/test/src/math/smoke/CanonicalizeTest.h index 7e2456f84705..3baf60c3140f 100644 --- a/libc/test/src/math/smoke/CanonicalizeTest.h +++ b/libc/test/src/math/smoke/CanonicalizeTest.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_CANONICALIZETEST_H #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_CANONICALIZETEST_H +#include "src/__support/FPUtil/FEnvImpl.h" #include "src/__support/FPUtil/FPBits.h" #include "src/__support/integer_literals.h" #include "test/UnitTest/FEnvSafeTest.h" diff --git a/libc/test/src/math/smoke/canonicalizef16_test.cpp b/libc/test/src/math/smoke/canonicalizef16_test.cpp new file mode 100644 index 000000000000..32a3d1756e4d --- /dev/null +++ b/libc/test/src/math/smoke/canonicalizef16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for canonicalizef16 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "CanonicalizeTest.h" + +#include "src/math/canonicalizef16.h" + +LIST_CANONICALIZE_TESTS(float16, LIBC_NAMESPACE::canonicalizef16) -- GitLab From 163cb1fc2fe4caa8306a18abdb0516870e4d7f3d Mon Sep 17 00:00:00 2001 From: EdJoPaTo Date: Wed, 5 Jun 2024 14:35:43 +0200 Subject: [PATCH 020/675] [llvm-cov] Add HTML dark theme support (#93080) Personally I use [cargo-llvm-cov](https://github.com/taiki-e/cargo-llvm-cov) which creates helpful HTML coverage reports, but they don't support a dynamic dark themes. I updated the styling to support both dark and bright color themes based on the browser preference. The bright theme should look similar to the current theme. I also improved some color contrasts (Firefox accessibility tool reported them) and ensured that line-number links keep their text-decoration. Things that both have `.tooltip` and `.red` look kinda odd as the coloring is now based on tinting with transparency. Given that the tooltip should always show 0 in such cases (otherwise it wouldn't be red) the tooltip could be removed there on the HTML generation, but that seemed out of scope for my style only change. --- .../tools/llvm-cov/SourceCoverageViewHTML.cpp | 76 ++++++++++--------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp b/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp index b93d8cb03530..d4b2ea3594fc 100644 --- a/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp +++ b/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp @@ -90,7 +90,7 @@ const char *BeginHeader = const char *CSSForCoverage = R"(.red { - background-color: #ffd0d0; + background-color: #f004; } .cyan { background-color: cyan; @@ -104,38 +104,35 @@ pre { } .source-name-title { padding: 5px 10px; - border-bottom: 1px solid #dbdbdb; - background-color: #eee; + border-bottom: 1px solid #8888; + background-color: #0002; line-height: 35px; } .centered { display: table; margin-left: left; margin-right: auto; - border: 1px solid #dbdbdb; + border: 1px solid #8888; border-radius: 3px; } .expansion-view { - background-color: rgba(0, 0, 0, 0); margin-left: 0px; margin-top: 5px; margin-right: 5px; margin-bottom: 5px; - border: 1px solid #dbdbdb; + border: 1px solid #8888; border-radius: 3px; } table { border-collapse: collapse; } .light-row { - background: #ffffff; - border: 1px solid #dbdbdb; + border: 1px solid #8888; border-left: none; border-right: none; } .light-row-bold { - background: #ffffff; - border: 1px solid #dbdbdb; + border: 1px solid #8888; border-left: none; border-right: none; font-weight: bold; @@ -149,48 +146,35 @@ table { } .column-entry-yellow { text-align: left; - background-color: #ffffd0; -} -.column-entry-yellow:hover, tr:hover .column-entry-yellow { - background-color: #fffff0; + background-color: #ff06; } .column-entry-red { text-align: left; - background-color: #ffd0d0; -} -.column-entry-red:hover, tr:hover .column-entry-red { - background-color: #fff0f0; + background-color: #f004; } .column-entry-gray { text-align: left; - background-color: #fbfbfb; -} -.column-entry-gray:hover, tr:hover .column-entry-gray { - background-color: #f0f0f0; + background-color: #fff4; } .column-entry-green { text-align: left; - background-color: #d0ffd0; -} -.column-entry-green:hover, tr:hover .column-entry-green { - background-color: #f0fff0; + background-color: #0f04; } .line-number { text-align: right; - color: #aaa; } .covered-line { text-align: right; - color: #0080ff; + color: #06d; } .uncovered-line { text-align: right; - color: #ff3300; + color: #d00; } .tooltip { position: relative; display: inline; - background-color: #b3e6ff; + background-color: #bef; text-decoration: none; } .tooltip span.tooltip-content { @@ -227,12 +211,13 @@ th, td { vertical-align: top; padding: 2px 8px; border-collapse: collapse; - border-right: solid 1px #eee; - border-left: solid 1px #eee; + border-right: 1px solid #8888; + border-left: 1px solid #8888; text-align: left; } td pre { display: inline-block; + text-decoration: inherit; } td:first-child { border-left: none; @@ -241,13 +226,34 @@ td:last-child { border-right: none; } tr:hover { - background-color: #f0f0f0; + background-color: #eee; } tr:last-child { border-bottom: none; } -tr:has(> td >a:target) > td.code > pre { - background-color: #ffa; +tr:has(> td >a:target) { + background-color: #50f6; +} +a { + color: inherit; +} +@media (prefers-color-scheme: dark) { + body { + background-color: #222; + color: whitesmoke; + } + tr:hover { + background-color: #111; + } + .covered-line { + color: #39f; + } + .uncovered-line { + color: #f55; + } + .tooltip { + background-color: #068; + } } )"; -- GitLab From 5a201415392bcd0e0b22d13e9aaae03ccf3043e6 Mon Sep 17 00:00:00 2001 From: Alex Bradbury Date: Wed, 5 Jun 2024 13:40:58 +0100 Subject: [PATCH 021/675] [LSR] Provide TTI hook to enable dropping solutions deemed to be unprofitable (#89924) introduced a flag to drop solutions if deemed unprofitable. As noted there, introducing a TTI hook enables backends to individually opt into this behaviour. This will be used by #89927. --- .../llvm/Analysis/TargetTransformInfo.h | 8 ++++++++ .../llvm/Analysis/TargetTransformInfoImpl.h | 2 ++ llvm/include/llvm/CodeGen/BasicTTIImpl.h | 4 ++++ llvm/lib/Analysis/TargetTransformInfo.cpp | 4 ++++ .../Transforms/Scalar/LoopStrengthReduce.cpp | 18 +++++++++++++++--- 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index cefce93f9e25..f55f21c94a85 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -740,6 +740,10 @@ public: /// When successful, makes the primary IV dead. bool shouldFoldTerminatingConditionAfterLSR() const; + /// Return true if LSR should drop a found solution if it's calculated to be + /// less profitable than the baseline. + bool shouldDropLSRSolutionIfLessProfitable() const; + /// \returns true if LSR should not optimize a chain that includes \p I. bool isProfitableLSRChainElement(Instruction *I) const; @@ -1864,6 +1868,7 @@ public: const TargetTransformInfo::LSRCost &C2) = 0; virtual bool isNumRegsMajorCostOfLSR() = 0; virtual bool shouldFoldTerminatingConditionAfterLSR() const = 0; + virtual bool shouldDropLSRSolutionIfLessProfitable() const = 0; virtual bool isProfitableLSRChainElement(Instruction *I) = 0; virtual bool canMacroFuseCmp() = 0; virtual bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, @@ -2337,6 +2342,9 @@ public: bool shouldFoldTerminatingConditionAfterLSR() const override { return Impl.shouldFoldTerminatingConditionAfterLSR(); } + bool shouldDropLSRSolutionIfLessProfitable() const override { + return Impl.shouldDropLSRSolutionIfLessProfitable(); + } bool isProfitableLSRChainElement(Instruction *I) override { return Impl.isProfitableLSRChainElement(I); } diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 9a57331d281d..7828bdc1f1f4 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -241,6 +241,8 @@ public: bool shouldFoldTerminatingConditionAfterLSR() const { return false; } + bool shouldDropLSRSolutionIfLessProfitable() const { return false; } + bool isProfitableLSRChainElement(Instruction *I) const { return false; } bool canMacroFuseCmp() const { return false; } diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 2091432d4fe2..ef4e0fd29768 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -399,6 +399,10 @@ public: shouldFoldTerminatingConditionAfterLSR(); } + bool shouldDropLSRSolutionIfLessProfitable() const { + return TargetTransformInfoImplBase::shouldDropLSRSolutionIfLessProfitable(); + } + bool isProfitableLSRChainElement(Instruction *I) { return TargetTransformInfoImplBase::isProfitableLSRChainElement(I); } diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index 82b6d7e7c483..7e721cbc87f3 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -427,6 +427,10 @@ bool TargetTransformInfo::shouldFoldTerminatingConditionAfterLSR() const { return TTIImpl->shouldFoldTerminatingConditionAfterLSR(); } +bool TargetTransformInfo::shouldDropLSRSolutionIfLessProfitable() const { + return TTIImpl->shouldDropLSRSolutionIfLessProfitable(); +} + bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const { return TTIImpl->isProfitableLSRChainElement(I); } diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp index 35a17d6060c9..73ed611e8de8 100644 --- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -193,8 +193,8 @@ static cl::opt AllowTerminatingConditionFoldingAfterLSR( "lsr-term-fold", cl::Hidden, cl::desc("Attempt to replace primary IV with other IV.")); -static cl::opt AllowDropSolutionIfLessProfitable( - "lsr-drop-solution", cl::Hidden, cl::init(false), +static cl::opt AllowDropSolutionIfLessProfitable( + "lsr-drop-solution", cl::Hidden, cl::desc("Attempt to drop solution if it is less profitable")); STATISTIC(NumTermFold, @@ -5250,8 +5250,20 @@ void LSRInstance::Solve(SmallVectorImpl &Solution) const { assert(Solution.size() == Uses.size() && "Malformed solution!"); + const bool EnableDropUnprofitableSolution = [&] { + switch (AllowDropSolutionIfLessProfitable) { + case cl::BOU_TRUE: + return true; + case cl::BOU_FALSE: + return false; + case cl::BOU_UNSET: + return TTI.shouldDropLSRSolutionIfLessProfitable(); + } + llvm_unreachable("Unhandled cl::boolOrDefault enum"); + }(); + if (BaselineCost.isLess(SolutionCost)) { - if (!AllowDropSolutionIfLessProfitable) + if (!EnableDropUnprofitableSolution) LLVM_DEBUG( dbgs() << "Baseline is more profitable than chosen solution, " "add option 'lsr-drop-solution' to drop LSR solution.\n"); -- GitLab From af76071ac078834ad0c4085e1c88198d3735fbd5 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Wed, 5 Jun 2024 13:59:12 +0100 Subject: [PATCH 022/675] [flang][CodeGen][NFC] Reduce TargetRewrite pass boilerplate (#94450) Tablegen can automatically generate the pass constructor. Tablegen will create a constructor for all of the pass options (not only the subset in the old constructor), but the pass options seem unused anyway. This pass does not require any modification to support alternative top-level ops. It walks all operations in the module. Functions have special handling (adding attributes, converting signatures) but this wouldn't make sense for top level operations in general. --- flang/include/flang/Optimizer/CodeGen/CGPasses.td | 1 - flang/include/flang/Optimizer/CodeGen/CodeGen.h | 12 ------------ flang/include/flang/Tools/CLOptions.inc | 5 ++--- flang/lib/Optimizer/CodeGen/TargetRewrite.cpp | 11 +---------- 4 files changed, 3 insertions(+), 26 deletions(-) diff --git a/flang/include/flang/Optimizer/CodeGen/CGPasses.td b/flang/include/flang/Optimizer/CodeGen/CGPasses.td index df042187b2a7..9a4d327b33ba 100644 --- a/flang/include/flang/Optimizer/CodeGen/CGPasses.td +++ b/flang/include/flang/Optimizer/CodeGen/CGPasses.td @@ -61,7 +61,6 @@ def TargetRewritePass : Pass<"target-rewrite", "mlir::ModuleOp"> { Certain abstractions in the FIR dialect need to be rewritten to reflect representations that may differ based on the target machine. }]; - let constructor = "::fir::createFirTargetRewritePass()"; let dependentDialects = [ "fir::FIROpsDialect", "mlir::func::FuncDialect", "mlir::DLTIDialect" ]; let options = [ diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h index 3063bf1c0e02..06961819bb19 100644 --- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h +++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h @@ -28,18 +28,6 @@ struct NameUniquer; #define GEN_PASS_DECL_BOXEDPROCEDUREPASS #include "flang/Optimizer/CodeGen/CGPasses.h.inc" -/// FirTargetRewritePass options. -struct TargetRewriteOptions { - bool noCharacterConversion{}; - bool noComplexConversion{}; - bool noStructConversion{}; -}; - -/// Prerequiste pass for code gen. Perform intermediate rewrites to tailor the -/// FIR for the chosen target. -std::unique_ptr> createFirTargetRewritePass( - const TargetRewriteOptions &options = TargetRewriteOptions()); - /// FIR to LLVM translation pass options. struct FIRToLLVMPassOptions { // Do not fail when type descriptors are not found when translating diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc index fb3ec75d4078..c5c35e9a6a33 100644 --- a/flang/include/flang/Tools/CLOptions.inc +++ b/flang/include/flang/Tools/CLOptions.inc @@ -182,9 +182,8 @@ inline void addCodeGenRewritePass(mlir::PassManager &pm, bool preserveDeclare) { } inline void addTargetRewritePass(mlir::PassManager &pm) { - addPassConditionally(pm, disableTargetRewrite, []() { - return fir::createFirTargetRewritePass(fir::TargetRewriteOptions{}); - }); + addPassConditionally(pm, disableTargetRewrite, + []() { return fir::createTargetRewritePass(); }); } inline mlir::LLVM::DIEmissionKind getEmissionKind( diff --git a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp index 616de78d0026..8199c5ef7fa2 100644 --- a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp +++ b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp @@ -76,11 +76,7 @@ struct FixupTy { /// idioms that are used for distinct target processor and ABI combinations. class TargetRewrite : public fir::impl::TargetRewritePassBase { public: - TargetRewrite(const fir::TargetRewriteOptions &options) { - noCharacterConversion = options.noCharacterConversion; - noComplexConversion = options.noComplexConversion; - noStructConversion = options.noStructConversion; - } + using TargetRewritePassBase::TargetRewritePassBase; void runOnOperation() override final { auto &context = getContext(); @@ -1255,8 +1251,3 @@ private: mlir::func::FuncOp stackRestoreFn = nullptr; }; } // namespace - -std::unique_ptr> -fir::createFirTargetRewritePass(const fir::TargetRewriteOptions &options) { - return std::make_unique(options); -} -- GitLab From 2d9b83750fea782276ec1f70157122b0b7d1856e Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Wed, 5 Jun 2024 15:13:29 +0200 Subject: [PATCH 023/675] [bazel] Add missing dep for __support_cpp_expected --- utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index aeb38a4a8729..f3809bd74814 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -319,6 +319,7 @@ libc_support_library( name = "__support_cpp_expected", hdrs = ["src/__support/CPP/expected.h"], deps = [ + ":__support_macros_attributes", ], ) -- GitLab From 42f4e505a38480b6a714b503dd946ffff31ae029 Mon Sep 17 00:00:00 2001 From: Erich Keane Date: Wed, 5 Jun 2024 06:21:48 -0700 Subject: [PATCH 024/675] [OpenACC] Loop construct basic Sema and AST work (#93742) This patch implements the 'loop' construct AST, as well as the basic appertainment rule. Additionally, it sets up the 'parent' compute construct, which is necessary for codegen/other diagnostics. A 'loop' can apply to a for or range-for loop, otherwise it has no other restrictions (though some of its clauses do). --- clang/include/clang-c/Index.h | 6 +- clang/include/clang/AST/RecursiveASTVisitor.h | 2 + clang/include/clang/AST/StmtOpenACC.h | 71 ++- clang/include/clang/AST/TextNodeDumper.h | 1 + .../clang/Basic/DiagnosticSemaKinds.td | 3 + clang/include/clang/Basic/StmtNodes.td | 1 + clang/include/clang/Sema/SemaOpenACC.h | 27 +- .../include/clang/Serialization/ASTBitCodes.h | 1 + clang/lib/AST/StmtOpenACC.cpp | 91 +++- clang/lib/AST/StmtPrinter.cpp | 13 + clang/lib/AST/StmtProfile.cpp | 8 + clang/lib/AST/TextNodeDumper.cpp | 7 + clang/lib/CodeGen/CGStmt.cpp | 3 + clang/lib/CodeGen/CodeGenFunction.h | 7 + clang/lib/Parse/ParseOpenACC.cpp | 8 +- clang/lib/Sema/SemaExceptionSpec.cpp | 1 + clang/lib/Sema/SemaOpenACC.cpp | 60 ++- clang/lib/Sema/TreeTransform.h | 44 +- clang/lib/Serialization/ASTReaderStmt.cpp | 11 + clang/lib/Serialization/ASTWriterStmt.cpp | 6 + clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 1 + .../AST/ast-print-openacc-loop-construct.cpp | 9 + clang/test/ParserOpenACC/parse-clauses.c | 410 ++++++++---------- clang/test/ParserOpenACC/parse-clauses.cpp | 6 +- clang/test/ParserOpenACC/parse-constructs.c | 3 +- .../compute-construct-async-clause.c | 3 +- .../compute-construct-attach-clause.c | 3 +- .../compute-construct-copy-clause.c | 9 +- .../compute-construct-copyin-clause.c | 9 +- .../compute-construct-copyout-clause.c | 9 +- .../compute-construct-create-clause.c | 9 +- .../compute-construct-default-clause.c | 6 +- .../compute-construct-deviceptr-clause.c | 3 +- .../compute-construct-firstprivate-clause.c | 3 +- .../SemaOpenACC/compute-construct-if-clause.c | 3 +- .../compute-construct-no_create-clause.c | 3 +- .../compute-construct-num_gangs-clause.c | 3 +- .../compute-construct-num_workers-clause.c | 3 +- .../compute-construct-present-clause.c | 3 +- .../compute-construct-self-clause.c | 3 +- .../compute-construct-vector_length-clause.c | 3 +- .../compute-construct-wait-clause.c | 3 +- clang/test/SemaOpenACC/loop-ast.cpp | 182 ++++++++ clang/test/SemaOpenACC/loop-loc-and-stmt.c | 38 ++ clang/test/SemaOpenACC/loop-loc-and-stmt.cpp | 80 ++++ clang/tools/libclang/CIndex.cpp | 9 + clang/tools/libclang/CXCursor.cpp | 3 + 47 files changed, 886 insertions(+), 304 deletions(-) create mode 100644 clang/test/AST/ast-print-openacc-loop-construct.cpp create mode 100644 clang/test/SemaOpenACC/loop-ast.cpp create mode 100644 clang/test/SemaOpenACC/loop-loc-and-stmt.c create mode 100644 clang/test/SemaOpenACC/loop-loc-and-stmt.cpp diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h index 365b607c7411..ce2282937f86 100644 --- a/clang/include/clang-c/Index.h +++ b/clang/include/clang-c/Index.h @@ -2150,7 +2150,11 @@ enum CXCursorKind { */ CXCursor_OpenACCComputeConstruct = 320, - CXCursor_LastStmt = CXCursor_OpenACCComputeConstruct, + /** OpenACC Loop Construct. + */ + CXCursor_OpenACCLoopConstruct = 321, + + CXCursor_LastStmt = CXCursor_OpenACCLoopConstruct, /** * Cursor that represents the translation unit itself. diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h index 99093aa17972..aa55e2e7e871 100644 --- a/clang/include/clang/AST/RecursiveASTVisitor.h +++ b/clang/include/clang/AST/RecursiveASTVisitor.h @@ -4000,6 +4000,8 @@ bool RecursiveASTVisitor::VisitOpenACCClauseList( DEF_TRAVERSE_STMT(OpenACCComputeConstruct, { TRY_TO(TraverseOpenACCAssociatedStmtConstruct(S)); }) +DEF_TRAVERSE_STMT(OpenACCLoopConstruct, + { TRY_TO(TraverseOpenACCAssociatedStmtConstruct(S)); }) // FIXME: look at the following tricky-seeming exprs to see if we // need to recurse on anything. These are ones that have methods diff --git a/clang/include/clang/AST/StmtOpenACC.h b/clang/include/clang/AST/StmtOpenACC.h index 04daf511f587..b3aea09be03d 100644 --- a/clang/include/clang/AST/StmtOpenACC.h +++ b/clang/include/clang/AST/StmtOpenACC.h @@ -113,6 +113,8 @@ public: return const_cast(this)->children(); } }; + +class OpenACCLoopConstruct; /// This class represents a compute construct, representing a 'Kind' of /// `parallel', 'serial', or 'kernel'. These constructs are associated with a /// 'structured block', defined as: @@ -165,6 +167,11 @@ class OpenACCComputeConstruct final } void setStructuredBlock(Stmt *S) { setAssociatedStmt(S); } + // Serialization helper function that searches the structured block for 'loop' + // constructs that should be associated with this, and sets their parent + // compute construct to this one. This isn't necessary normally, since we have + // the ability to record the state during parsing. + void findAndSetChildLoops(); public: static bool classof(const Stmt *T) { @@ -176,12 +183,74 @@ public: static OpenACCComputeConstruct * Create(const ASTContext &C, OpenACCDirectiveKind K, SourceLocation BeginLoc, SourceLocation DirectiveLoc, SourceLocation EndLoc, - ArrayRef Clauses, Stmt *StructuredBlock); + ArrayRef Clauses, Stmt *StructuredBlock, + ArrayRef AssociatedLoopConstructs); Stmt *getStructuredBlock() { return getAssociatedStmt(); } const Stmt *getStructuredBlock() const { return const_cast(this)->getStructuredBlock(); } }; +/// This class represents a 'loop' construct. The 'loop' construct applies to a +/// 'for' loop (or range-for loop), and is optionally associated with a Compute +/// Construct. +class OpenACCLoopConstruct final + : public OpenACCAssociatedStmtConstruct, + public llvm::TrailingObjects { + // The compute construct this loop is associated with, or nullptr if this is + // an orphaned loop construct, or if it hasn't been set yet. Because we + // construct the directives at the end of their statement, the 'parent' + // construct is not yet available at the time of construction, so this needs + // to be set 'later'. + const OpenACCComputeConstruct *ParentComputeConstruct = nullptr; + + friend class ASTStmtWriter; + friend class ASTStmtReader; + friend class ASTContext; + friend class OpenACCComputeConstruct; + + OpenACCLoopConstruct(unsigned NumClauses); + + OpenACCLoopConstruct(SourceLocation Start, SourceLocation DirLoc, + SourceLocation End, + ArrayRef Clauses, Stmt *Loop); + void setLoop(Stmt *Loop); + + void setParentComputeConstruct(OpenACCComputeConstruct *CC) { + assert(!ParentComputeConstruct && "Parent already set?"); + ParentComputeConstruct = CC; + } + +public: + static bool classof(const Stmt *T) { + return T->getStmtClass() == OpenACCLoopConstructClass; + } + + static OpenACCLoopConstruct *CreateEmpty(const ASTContext &C, + unsigned NumClauses); + + static OpenACCLoopConstruct * + Create(const ASTContext &C, SourceLocation BeginLoc, SourceLocation DirLoc, + SourceLocation EndLoc, ArrayRef Clauses, + Stmt *Loop); + + Stmt *getLoop() { return getAssociatedStmt(); } + const Stmt *getLoop() const { + return const_cast(this)->getLoop(); + } + + /// OpenACC 3.3 2.9: + /// An orphaned loop construct is a loop construct that is not lexically + /// enclosed within a compute construct. The parent compute construct of a + /// loop construct is the nearest compute construct that lexically contains + /// the loop construct. + bool isOrphanedLoopConstruct() const { + return ParentComputeConstruct == nullptr; + } + const OpenACCComputeConstruct *getParentComputeConstruct() const { + return ParentComputeConstruct; + } +}; } // namespace clang #endif // LLVM_CLANG_AST_STMTOPENACC_H diff --git a/clang/include/clang/AST/TextNodeDumper.h b/clang/include/clang/AST/TextNodeDumper.h index caa33abd99e4..abfafcaef271 100644 --- a/clang/include/clang/AST/TextNodeDumper.h +++ b/clang/include/clang/AST/TextNodeDumper.h @@ -408,6 +408,7 @@ public: VisitLifetimeExtendedTemporaryDecl(const LifetimeExtendedTemporaryDecl *D); void VisitHLSLBufferDecl(const HLSLBufferDecl *D); void VisitOpenACCConstructStmt(const OpenACCConstructStmt *S); + void VisitOpenACCLoopConstruct(const OpenACCLoopConstruct *S); }; } // namespace clang diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index e34eb692941b..50332966a7e3 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -12413,6 +12413,9 @@ def err_acc_reduction_composite_type def err_acc_reduction_composite_member_type :Error< "OpenACC 'reduction' composite variable must not have non-scalar field">; def note_acc_reduction_composite_member_loc : Note<"invalid field is here">; +def err_acc_loop_not_for_loop + : Error<"OpenACC 'loop' construct can only be applied to a 'for' loop">; +def note_acc_construct_here : Note<"'%0' construct is here">; // AMDGCN builtins diagnostics def err_amdgcn_global_load_lds_size_invalid_value : Error<"invalid size value">; diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td index 305f19daa4a9..6ca08abdb14f 100644 --- a/clang/include/clang/Basic/StmtNodes.td +++ b/clang/include/clang/Basic/StmtNodes.td @@ -302,3 +302,4 @@ def OpenACCConstructStmt : StmtNode; def OpenACCAssociatedStmtConstruct : StmtNode; def OpenACCComputeConstruct : StmtNode; +def OpenACCLoopConstruct : StmtNode; diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h index 66144de4340a..a5f2a8bf7465 100644 --- a/clang/include/clang/Sema/SemaOpenACC.h +++ b/clang/include/clang/Sema/SemaOpenACC.h @@ -15,6 +15,7 @@ #define LLVM_CLANG_SEMA_SEMAOPENACC_H #include "clang/AST/DeclGroup.h" +#include "clang/AST/StmtOpenACC.h" #include "clang/Basic/OpenACCKinds.h" #include "clang/Basic/SourceLocation.h" #include "clang/Sema/Ownership.h" @@ -25,6 +26,15 @@ namespace clang { class OpenACCClause; class SemaOpenACC : public SemaBase { +private: + /// A collection of loop constructs in the compute construct scope that + /// haven't had their 'parent' compute construct set yet. Entires will only be + /// made to this list in the case where we know the loop isn't an orphan. + llvm::SmallVector ParentlessLoopConstructs; + /// Whether we are inside of a compute construct, and should add loops to the + /// above collection. + bool InsideComputeConstruct = false; + public: // Redeclaration of the version in OpenACCClause.h. using DeviceTypeArgument = std::pair; @@ -394,7 +404,8 @@ public: bool ActOnStartDeclDirective(OpenACCDirectiveKind K, SourceLocation StartLoc); /// Called when we encounter an associated statement for our construct, this /// should check legality of the statement as it appertains to this Construct. - StmtResult ActOnAssociatedStmt(OpenACCDirectiveKind K, StmtResult AssocStmt); + StmtResult ActOnAssociatedStmt(SourceLocation DirectiveLoc, + OpenACCDirectiveKind K, StmtResult AssocStmt); /// Called after the directive has been completely parsed, including the /// declaration group or associated statement. @@ -431,6 +442,20 @@ public: Expr *LowerBound, SourceLocation ColonLocFirst, Expr *Length, SourceLocation RBLoc); + + /// Helper type for the registration/assignment of constructs that need to + /// 'know' about their parent constructs and hold a reference to them, such as + /// Loop needing its parent construct. + class AssociatedStmtRAII { + SemaOpenACC &SemaRef; + bool WasInsideComputeConstruct; + OpenACCDirectiveKind DirKind; + llvm::SmallVector ParentlessLoopConstructs; + + public: + AssociatedStmtRAII(SemaOpenACC &, OpenACCDirectiveKind); + ~AssociatedStmtRAII(); + }; }; } // namespace clang diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index fe1bd47348be..f59ff6af4c76 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -1946,6 +1946,7 @@ enum StmtCode { // OpenACC Constructs STMT_OPENACC_COMPUTE_CONSTRUCT, + STMT_OPENACC_LOOP_CONSTRUCT, }; /// The kinds of designators that can occur in a diff --git a/clang/lib/AST/StmtOpenACC.cpp b/clang/lib/AST/StmtOpenACC.cpp index 47899b344c97..2d864a288579 100644 --- a/clang/lib/AST/StmtOpenACC.cpp +++ b/clang/lib/AST/StmtOpenACC.cpp @@ -12,6 +12,8 @@ #include "clang/AST/StmtOpenACC.h" #include "clang/AST/ASTContext.h" +#include "clang/AST/RecursiveASTVisitor.h" +#include "clang/AST/StmtCXX.h" using namespace clang; OpenACCComputeConstruct * @@ -26,11 +28,98 @@ OpenACCComputeConstruct::CreateEmpty(const ASTContext &C, unsigned NumClauses) { OpenACCComputeConstruct *OpenACCComputeConstruct::Create( const ASTContext &C, OpenACCDirectiveKind K, SourceLocation BeginLoc, SourceLocation DirLoc, SourceLocation EndLoc, - ArrayRef Clauses, Stmt *StructuredBlock) { + ArrayRef Clauses, Stmt *StructuredBlock, + ArrayRef AssociatedLoopConstructs) { void *Mem = C.Allocate( OpenACCComputeConstruct::totalSizeToAlloc( Clauses.size())); auto *Inst = new (Mem) OpenACCComputeConstruct(K, BeginLoc, DirLoc, EndLoc, Clauses, StructuredBlock); + + llvm::for_each(AssociatedLoopConstructs, [&](OpenACCLoopConstruct *C) { + C->setParentComputeConstruct(Inst); + }); + + return Inst; +} + +void OpenACCComputeConstruct::findAndSetChildLoops() { + struct LoopConstructFinder : RecursiveASTVisitor { + OpenACCComputeConstruct *Construct = nullptr; + + LoopConstructFinder(OpenACCComputeConstruct *Construct) + : Construct(Construct) {} + + bool TraverseOpenACCComputeConstruct(OpenACCComputeConstruct *C) { + // Stop searching if we find a compute construct. + return true; + } + bool TraverseOpenACCLoopConstruct(OpenACCLoopConstruct *C) { + // Stop searching if we find a loop construct, after taking ownership of + // it. + C->setParentComputeConstruct(Construct); + return true; + } + }; + + LoopConstructFinder f(this); + f.TraverseStmt(getAssociatedStmt()); +} + +OpenACCLoopConstruct::OpenACCLoopConstruct(unsigned NumClauses) + : OpenACCAssociatedStmtConstruct( + OpenACCLoopConstructClass, OpenACCDirectiveKind::Loop, + SourceLocation{}, SourceLocation{}, SourceLocation{}, + /*AssociatedStmt=*/nullptr) { + std::uninitialized_value_construct( + getTrailingObjects(), + getTrailingObjects() + NumClauses); + setClauseList( + MutableArrayRef(getTrailingObjects(), NumClauses)); +} + +OpenACCLoopConstruct::OpenACCLoopConstruct( + SourceLocation Start, SourceLocation DirLoc, SourceLocation End, + ArrayRef Clauses, Stmt *Loop) + : OpenACCAssociatedStmtConstruct(OpenACCLoopConstructClass, + OpenACCDirectiveKind::Loop, Start, DirLoc, + End, Loop) { + // accept 'nullptr' for the loop. This is diagnosed somewhere, but this gives + // us some level of AST fidelity in the error case. + assert((Loop == nullptr || isa(Loop)) && + "Associated Loop not a for loop?"); + // Initialize the trailing storage. + std::uninitialized_copy(Clauses.begin(), Clauses.end(), + getTrailingObjects()); + + setClauseList(MutableArrayRef(getTrailingObjects(), + Clauses.size())); +} + +void OpenACCLoopConstruct::setLoop(Stmt *Loop) { + assert((isa(Loop)) && + "Associated Loop not a for loop?"); + setAssociatedStmt(Loop); +} + +OpenACCLoopConstruct *OpenACCLoopConstruct::CreateEmpty(const ASTContext &C, + unsigned NumClauses) { + void *Mem = + C.Allocate(OpenACCLoopConstruct::totalSizeToAlloc( + NumClauses)); + auto *Inst = new (Mem) OpenACCLoopConstruct(NumClauses); + return Inst; +} + +OpenACCLoopConstruct * +OpenACCLoopConstruct::Create(const ASTContext &C, SourceLocation BeginLoc, + SourceLocation DirLoc, SourceLocation EndLoc, + ArrayRef Clauses, + Stmt *Loop) { + void *Mem = + C.Allocate(OpenACCLoopConstruct::totalSizeToAlloc( + Clauses.size())); + auto *Inst = + new (Mem) OpenACCLoopConstruct(BeginLoc, DirLoc, EndLoc, Clauses, Loop); return Inst; } diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp index be2d5a2eb6b4..7e030e055126 100644 --- a/clang/lib/AST/StmtPrinter.cpp +++ b/clang/lib/AST/StmtPrinter.cpp @@ -1156,6 +1156,19 @@ void StmtPrinter::VisitOpenACCComputeConstruct(OpenACCComputeConstruct *S) { PrintStmt(S->getStructuredBlock()); } +void StmtPrinter::VisitOpenACCLoopConstruct(OpenACCLoopConstruct *S) { + Indent() << "#pragma acc loop"; + + if (!S->clauses().empty()) { + OS << ' '; + OpenACCClausePrinter Printer(OS, Policy); + Printer.VisitClauseList(S->clauses()); + } + OS << '\n'; + + PrintStmt(S->getLoop()); +} + //===----------------------------------------------------------------------===// // Expr printing methods. //===----------------------------------------------------------------------===// diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp index 00b8c43af035..6d9a76120cfe 100644 --- a/clang/lib/AST/StmtProfile.cpp +++ b/clang/lib/AST/StmtProfile.cpp @@ -2605,6 +2605,14 @@ void StmtProfiler::VisitOpenACCComputeConstruct( P.VisitOpenACCClauseList(S->clauses()); } +void StmtProfiler::VisitOpenACCLoopConstruct(const OpenACCLoopConstruct *S) { + // VisitStmt handles children, so the Loop is handled. + VisitStmt(S); + + OpenACCClauseProfiler P{*this}; + P.VisitOpenACCClauseList(S->clauses()); +} + void Stmt::Profile(llvm::FoldingSetNodeID &ID, const ASTContext &Context, bool Canonical, bool ProfileLambdaExpr) const { StmtProfilerWithPointers Profiler(ID, Context, Canonical, ProfileLambdaExpr); diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp index 0e0e0a86f5cf..b2bf259ec24e 100644 --- a/clang/lib/AST/TextNodeDumper.cpp +++ b/clang/lib/AST/TextNodeDumper.cpp @@ -2869,3 +2869,10 @@ void TextNodeDumper::VisitHLSLBufferDecl(const HLSLBufferDecl *D) { void TextNodeDumper::VisitOpenACCConstructStmt(const OpenACCConstructStmt *S) { OS << " " << S->getDirectiveKind(); } +void TextNodeDumper::VisitOpenACCLoopConstruct(const OpenACCLoopConstruct *S) { + + if (S->isOrphanedLoopConstruct()) + OS << " "; + else + OS << " parent: " << S->getParentComputeConstruct(); +} diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp index 99daaa14cf3f..41ac511c52f5 100644 --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -442,6 +442,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef Attrs) { case Stmt::OpenACCComputeConstructClass: EmitOpenACCComputeConstruct(cast(*S)); break; + case Stmt::OpenACCLoopConstructClass: + EmitOpenACCLoopConstruct(cast(*S)); + break; } } diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index 45585361a4fc..5739fbaaa919 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -4062,6 +4062,13 @@ public: EmitStmt(S.getStructuredBlock()); } + void EmitOpenACCLoopConstruct(const OpenACCLoopConstruct &S) { + // TODO OpenACC: Implement this. It is currently implemented as a 'no-op', + // simply emitting its loop, but in the future we will implement + // some sort of IR. + EmitStmt(S.getLoop()); + } + //===--------------------------------------------------------------------===// // LValue Expression Emission //===--------------------------------------------------------------------===// diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp index 63afc18783a1..c7b6763b4dbd 100644 --- a/clang/lib/Parse/ParseOpenACC.cpp +++ b/clang/lib/Parse/ParseOpenACC.cpp @@ -571,6 +571,7 @@ bool doesDirectiveHaveAssociatedStmt(OpenACCDirectiveKind DirKind) { case OpenACCDirectiveKind::Parallel: case OpenACCDirectiveKind::Serial: case OpenACCDirectiveKind::Kernels: + case OpenACCDirectiveKind::Loop: return true; } llvm_unreachable("Unhandled directive->assoc stmt"); @@ -1447,13 +1448,14 @@ StmtResult Parser::ParseOpenACCDirectiveStmt() { return StmtError(); StmtResult AssocStmt; - + SemaOpenACC::AssociatedStmtRAII AssocStmtRAII(getActions().OpenACC(), + DirInfo.DirKind); if (doesDirectiveHaveAssociatedStmt(DirInfo.DirKind)) { ParsingOpenACCDirectiveRAII DirScope(*this, /*Value=*/false); ParseScope ACCScope(this, getOpenACCScopeFlags(DirInfo.DirKind)); - AssocStmt = getActions().OpenACC().ActOnAssociatedStmt(DirInfo.DirKind, - ParseStatement()); + AssocStmt = getActions().OpenACC().ActOnAssociatedStmt( + DirInfo.StartLoc, DirInfo.DirKind, ParseStatement()); } return getActions().OpenACC().ActOnEndStmtDirective( diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp index 41bf273d12f2..17acfca6b011 100644 --- a/clang/lib/Sema/SemaExceptionSpec.cpp +++ b/clang/lib/Sema/SemaExceptionSpec.cpp @@ -1425,6 +1425,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) { // Most statements can throw if any substatement can throw. case Stmt::OpenACCComputeConstructClass: + case Stmt::OpenACCLoopConstructClass: case Stmt::AttributedStmtClass: case Stmt::BreakStmtClass: case Stmt::CapturedStmtClass: diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp index 5b4d01860c0b..92da218010c9 100644 --- a/clang/lib/Sema/SemaOpenACC.cpp +++ b/clang/lib/Sema/SemaOpenACC.cpp @@ -33,6 +33,7 @@ bool diagnoseConstructAppertainment(SemaOpenACC &S, OpenACCDirectiveKind K, case OpenACCDirectiveKind::Parallel: case OpenACCDirectiveKind::Serial: case OpenACCDirectiveKind::Kernels: + case OpenACCDirectiveKind::Loop: if (!IsStmt) return S.Diag(StartLoc, diag::err_acc_construct_appertainment) << K; break; @@ -370,6 +371,30 @@ bool checkValidAfterDeviceType( SemaOpenACC::SemaOpenACC(Sema &S) : SemaBase(S) {} +SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(SemaOpenACC &S, + OpenACCDirectiveKind DK) + : SemaRef(S), WasInsideComputeConstruct(S.InsideComputeConstruct), + DirKind(DK) { + // Compute constructs end up taking their 'loop'. + if (DirKind == OpenACCDirectiveKind::Parallel || + DirKind == OpenACCDirectiveKind::Serial || + DirKind == OpenACCDirectiveKind::Kernels) { + SemaRef.InsideComputeConstruct = true; + SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs); + } +} + +SemaOpenACC::AssociatedStmtRAII::~AssociatedStmtRAII() { + SemaRef.InsideComputeConstruct = WasInsideComputeConstruct; + if (DirKind == OpenACCDirectiveKind::Parallel || + DirKind == OpenACCDirectiveKind::Serial || + DirKind == OpenACCDirectiveKind::Kernels) { + assert(SemaRef.ParentlessLoopConstructs.empty() && + "Didn't consume loop construct list?"); + SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs); + } +} + OpenACCClause * SemaOpenACC::ActOnClause(ArrayRef ExistingClauses, OpenACCParsedClause &Clause) { @@ -927,6 +952,7 @@ void SemaOpenACC::ActOnConstruct(OpenACCDirectiveKind K, case OpenACCDirectiveKind::Parallel: case OpenACCDirectiveKind::Serial: case OpenACCDirectiveKind::Kernels: + case OpenACCDirectiveKind::Loop: // Nothing to do here, there is no real legalization that needs to happen // here as these constructs do not take any arguments. break; @@ -1348,16 +1374,34 @@ StmtResult SemaOpenACC::ActOnEndStmtDirective(OpenACCDirectiveKind K, return StmtError(); case OpenACCDirectiveKind::Parallel: case OpenACCDirectiveKind::Serial: - case OpenACCDirectiveKind::Kernels: - // TODO OpenACC: Add clauses to the construct here. - return OpenACCComputeConstruct::Create( + case OpenACCDirectiveKind::Kernels: { + auto *ComputeConstruct = OpenACCComputeConstruct::Create( getASTContext(), K, StartLoc, DirLoc, EndLoc, Clauses, + AssocStmt.isUsable() ? AssocStmt.get() : nullptr, + ParentlessLoopConstructs); + + ParentlessLoopConstructs.clear(); + return ComputeConstruct; + } + case OpenACCDirectiveKind::Loop: { + auto *LoopConstruct = OpenACCLoopConstruct::Create( + getASTContext(), StartLoc, DirLoc, EndLoc, Clauses, AssocStmt.isUsable() ? AssocStmt.get() : nullptr); + + // If we are in the scope of a compute construct, add this to the list of + // loop constructs that need assigning to the next closing compute + // construct. + if (InsideComputeConstruct) + ParentlessLoopConstructs.push_back(LoopConstruct); + + return LoopConstruct; + } } llvm_unreachable("Unhandled case in directive handling?"); } -StmtResult SemaOpenACC::ActOnAssociatedStmt(OpenACCDirectiveKind K, +StmtResult SemaOpenACC::ActOnAssociatedStmt(SourceLocation DirectiveLoc, + OpenACCDirectiveKind K, StmtResult AssocStmt) { switch (K) { default: @@ -1375,6 +1419,14 @@ StmtResult SemaOpenACC::ActOnAssociatedStmt(OpenACCDirectiveKind K, // an interpretation of it is to allow this and treat the initializer as // the 'structured block'. return AssocStmt; + case OpenACCDirectiveKind::Loop: + if (AssocStmt.isUsable() && + !isa(AssocStmt.get())) { + Diag(AssocStmt.get()->getBeginLoc(), diag::err_acc_loop_not_for_loop); + Diag(DirectiveLoc, diag::note_acc_construct_here) << K; + return StmtError(); + } + return AssocStmt; } llvm_unreachable("Invalid associated statement application"); } diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h index 70603ba6c271..07f995edaf3d 100644 --- a/clang/lib/Sema/TreeTransform.h +++ b/clang/lib/Sema/TreeTransform.h @@ -4041,6 +4041,15 @@ public: EndLoc, Clauses, StrBlock); } + StmtResult RebuildOpenACCLoopConstruct(SourceLocation BeginLoc, + SourceLocation DirLoc, + SourceLocation EndLoc, + ArrayRef Clauses, + StmtResult Loop) { + return getSema().OpenACC().ActOnEndStmtDirective( + OpenACCDirectiveKind::Loop, BeginLoc, DirLoc, EndLoc, Clauses, Loop); + } + private: TypeLoc TransformTypeInObjectScope(TypeLoc TL, QualType ObjectType, @@ -11541,8 +11550,6 @@ template StmtResult TreeTransform::TransformOpenACCComputeConstruct( OpenACCComputeConstruct *C) { getSema().OpenACC().ActOnConstruct(C->getDirectiveKind(), C->getBeginLoc()); - // FIXME: When implementing this for constructs that can take arguments, we - // should do Sema for them here. if (getSema().OpenACC().ActOnStartStmtDirective(C->getDirectiveKind(), C->getBeginLoc())) @@ -11551,17 +11558,44 @@ StmtResult TreeTransform::TransformOpenACCComputeConstruct( llvm::SmallVector TransformedClauses = getDerived().TransformOpenACCClauseList(C->getDirectiveKind(), C->clauses()); - // Transform Structured Block. + SemaOpenACC::AssociatedStmtRAII AssocStmtRAII(getSema().OpenACC(), + C->getDirectiveKind()); StmtResult StrBlock = getDerived().TransformStmt(C->getStructuredBlock()); - StrBlock = - getSema().OpenACC().ActOnAssociatedStmt(C->getDirectiveKind(), StrBlock); + StrBlock = getSema().OpenACC().ActOnAssociatedStmt( + C->getBeginLoc(), C->getDirectiveKind(), StrBlock); return getDerived().RebuildOpenACCComputeConstruct( C->getDirectiveKind(), C->getBeginLoc(), C->getDirectiveLoc(), C->getEndLoc(), TransformedClauses, StrBlock); } +template +StmtResult +TreeTransform::TransformOpenACCLoopConstruct(OpenACCLoopConstruct *C) { + + getSema().OpenACC().ActOnConstruct(C->getDirectiveKind(), C->getBeginLoc()); + + if (getSema().OpenACC().ActOnStartStmtDirective(C->getDirectiveKind(), + C->getBeginLoc())) + return StmtError(); + + llvm::SmallVector TransformedClauses = + getDerived().TransformOpenACCClauseList(C->getDirectiveKind(), + C->clauses()); + + // Transform Loop. + SemaOpenACC::AssociatedStmtRAII AssocStmtRAII(getSema().OpenACC(), + C->getDirectiveKind()); + StmtResult Loop = getDerived().TransformStmt(C->getLoop()); + Loop = getSema().OpenACC().ActOnAssociatedStmt(C->getBeginLoc(), + C->getDirectiveKind(), Loop); + + return getDerived().RebuildOpenACCLoopConstruct( + C->getBeginLoc(), C->getDirectiveLoc(), C->getEndLoc(), + TransformedClauses, Loop); +} + //===----------------------------------------------------------------------===// // Expression transformation //===----------------------------------------------------------------------===// diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp index bea2b9498910..67ef17025191 100644 --- a/clang/lib/Serialization/ASTReaderStmt.cpp +++ b/clang/lib/Serialization/ASTReaderStmt.cpp @@ -2810,6 +2810,12 @@ void ASTStmtReader::VisitOpenACCAssociatedStmtConstruct( void ASTStmtReader::VisitOpenACCComputeConstruct(OpenACCComputeConstruct *S) { VisitStmt(S); VisitOpenACCAssociatedStmtConstruct(S); + S->findAndSetChildLoops(); +} + +void ASTStmtReader::VisitOpenACCLoopConstruct(OpenACCLoopConstruct *S) { + VisitStmt(S); + VisitOpenACCAssociatedStmtConstruct(S); } //===----------------------------------------------------------------------===// @@ -4235,6 +4241,11 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = OpenACCComputeConstruct::CreateEmpty(Context, NumClauses); break; } + case STMT_OPENACC_LOOP_CONSTRUCT: { + unsigned NumClauses = Record[ASTStmtReader::NumStmtFields]; + S = OpenACCLoopConstruct::CreateEmpty(Context, NumClauses); + break; + } case EXPR_REQUIRES: unsigned numLocalParameters = Record[ASTStmtReader::NumExprFields]; unsigned numRequirement = Record[ASTStmtReader::NumExprFields + 1]; diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp index 3c586b270fbf..1a98e30e0f89 100644 --- a/clang/lib/Serialization/ASTWriterStmt.cpp +++ b/clang/lib/Serialization/ASTWriterStmt.cpp @@ -2863,6 +2863,12 @@ void ASTStmtWriter::VisitOpenACCComputeConstruct(OpenACCComputeConstruct *S) { Code = serialization::STMT_OPENACC_COMPUTE_CONSTRUCT; } +void ASTStmtWriter::VisitOpenACCLoopConstruct(OpenACCLoopConstruct *S) { + VisitStmt(S); + VisitOpenACCAssociatedStmtConstruct(S); + Code = serialization::STMT_OPENACC_LOOP_CONSTRUCT; +} + //===----------------------------------------------------------------------===// // ASTWriter Implementation //===----------------------------------------------------------------------===// diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp index 793f3a63ea29..290d96611d46 100644 --- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp +++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp @@ -1822,6 +1822,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred, case Stmt::OMPTargetParallelGenericLoopDirectiveClass: case Stmt::CapturedStmtClass: case Stmt::OpenACCComputeConstructClass: + case Stmt::OpenACCLoopConstructClass: case Stmt::OMPUnrollDirectiveClass: case Stmt::OMPMetaDirectiveClass: { const ExplodedNode *node = Bldr.generateSink(S, Pred, Pred->getState()); diff --git a/clang/test/AST/ast-print-openacc-loop-construct.cpp b/clang/test/AST/ast-print-openacc-loop-construct.cpp new file mode 100644 index 000000000000..21c92b17317e --- /dev/null +++ b/clang/test/AST/ast-print-openacc-loop-construct.cpp @@ -0,0 +1,9 @@ +// RUN: %clang_cc1 -fopenacc -Wno-openacc-deprecated-clause-alias -ast-print %s -o - | FileCheck %s + +void foo() { +// CHECK: #pragma acc loop +// CHECK-NEXT: for (;;) +// CHECK-NEXT: ; +#pragma acc loop + for(;;); +} diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c index 49e749feb2ec..cb118f69fb44 100644 --- a/clang/test/ParserOpenACC/parse-clauses.c +++ b/clang/test/ParserOpenACC/parse-clauses.c @@ -37,23 +37,23 @@ void func() { // expected-warning@+1{{OpenACC construct 'host_data' not yet implemented, pragma ignored}} #pragma acc host_data if_present, if_present - // expected-warning@+4{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+3{{OpenACC clause 'independent' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'auto' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+3{{OpenACC clause 'seq' not yet implemented, clause ignored}} + // expected-warning@+2{{OpenACC clause 'independent' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'auto' not yet implemented, clause ignored}} #pragma acc loop seq independent auto + for(;;){} - // expected-warning@+4{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+3{{OpenACC clause 'independent' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'auto' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+3{{OpenACC clause 'seq' not yet implemented, clause ignored}} + // expected-warning@+2{{OpenACC clause 'independent' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'auto' not yet implemented, clause ignored}} #pragma acc loop seq, independent auto + for(;;){} - // expected-warning@+4{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+3{{OpenACC clause 'independent' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'auto' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+3{{OpenACC clause 'seq' not yet implemented, clause ignored}} + // expected-warning@+2{{OpenACC clause 'independent' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'auto' not yet implemented, clause ignored}} #pragma acc loop seq independent, auto + for(;;){} // expected-warning@+4{{OpenACC clause 'seq' not yet implemented, clause ignored}} // expected-warning@+3{{OpenACC clause 'independent' not yet implemented, clause ignored}} @@ -67,65 +67,57 @@ void func() { // expected-warning@+2{{OpenACC clause 'auto' not yet implemented, clause ignored}} // expected-warning@+1{{OpenACC construct 'serial loop' not yet implemented, pragma ignored}} #pragma acc serial loop seq, independent auto - {} + for(;;){} // expected-warning@+4{{OpenACC clause 'seq' not yet implemented, clause ignored}} // expected-warning@+3{{OpenACC clause 'independent' not yet implemented, clause ignored}} // expected-warning@+2{{OpenACC clause 'auto' not yet implemented, clause ignored}} // expected-warning@+1{{OpenACC construct 'parallel loop' not yet implemented, pragma ignored}} #pragma acc parallel loop seq independent, auto - {} + for(;;){} - // expected-error@+2{{expected identifier}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+1{{expected identifier}} #pragma acc loop , seq + for(;;){} - // expected-error@+3{{expected identifier}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc loop seq, + for(;;){} - // expected-error@+2{{expected '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+1{{expected '('}} #pragma acc loop collapse for(;;){} - // expected-error@+2{{expected expression}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+1{{expected expression}} #pragma acc loop collapse() for(;;){} - // expected-error@+3{{invalid tag 'unknown' on 'collapse' clause}} - // expected-error@+2{{expected expression}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{invalid tag 'unknown' on 'collapse' clause}} + // expected-error@+1{{expected expression}} #pragma acc loop collapse(unknown:) for(;;){} - // expected-error@+2{{expected expression}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+1{{expected expression}} #pragma acc loop collapse(force:) for(;;){} - // expected-error@+3{{invalid tag 'unknown' on 'collapse' clause}} - // expected-warning@+2{{OpenACC clause 'collapse' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{invalid tag 'unknown' on 'collapse' clause}} + // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented, clause ignored}} #pragma acc loop collapse(unknown:5) for(;;){} - // expected-warning@+2{{OpenACC clause 'collapse' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented, clause ignored}} #pragma acc loop collapse(force:5) for(;;){} - // expected-warning@+2{{OpenACC clause 'collapse' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented, clause ignored}} #pragma acc loop collapse(5) for(;;){} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop collapse(5, 6) for(;;){} } @@ -989,108 +981,108 @@ void IntExprParsing() { #pragma acc set default_async(returns_int()) - // expected-warning@+2{{OpenACC clause 'vector' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'vector' not yet implemented, clause ignored}} #pragma acc loop vector - // expected-error@+2{{expected expression}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+1{{expected expression}} #pragma acc loop vector() - // expected-error@+3{{invalid tag 'invalid' on 'vector' clause}} - // expected-error@+2{{expected expression}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{invalid tag 'invalid' on 'vector' clause}} + // expected-error@+1{{expected expression}} #pragma acc loop vector(invalid:) - // expected-error@+3{{invalid tag 'invalid' on 'vector' clause}} - // expected-warning@+2{{OpenACC clause 'vector' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{invalid tag 'invalid' on 'vector' clause}} + // expected-warning@+1{{OpenACC clause 'vector' not yet implemented, clause ignored}} #pragma acc loop vector(invalid:5) - // expected-error@+2{{expected expression}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+1{{expected expression}} #pragma acc loop vector(length:) - // expected-error@+3{{invalid tag 'num' on 'vector' clause}} - // expected-error@+2{{expected expression}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{invalid tag 'num' on 'vector' clause}} + // expected-error@+1{{expected expression}} #pragma acc loop vector(num:) - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop vector(5, 4) - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop vector(length:6,4) - // expected-error@+4{{invalid tag 'num' on 'vector' clause}} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+3{{invalid tag 'num' on 'vector' clause}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop vector(num:6,4) - // expected-warning@+2{{OpenACC clause 'vector' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-warning@+1{{OpenACC clause 'vector' not yet implemented, clause ignored}} #pragma acc loop vector(5) - // expected-error@+3{{invalid tag 'num' on 'vector' clause}} - // expected-warning@+2{{OpenACC clause 'vector' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{invalid tag 'num' on 'vector' clause}} + // expected-warning@+1{{OpenACC clause 'vector' not yet implemented, clause ignored}} #pragma acc loop vector(num:5) - // expected-warning@+2{{OpenACC clause 'vector' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-warning@+1{{OpenACC clause 'vector' not yet implemented, clause ignored}} #pragma acc loop vector(length:5) - // expected-warning@+2{{OpenACC clause 'vector' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-warning@+1{{OpenACC clause 'vector' not yet implemented, clause ignored}} #pragma acc loop vector(returns_int()) - // expected-warning@+2{{OpenACC clause 'vector' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-warning@+1{{OpenACC clause 'vector' not yet implemented, clause ignored}} #pragma acc loop vector(length:returns_int()) + for(;;); - // expected-warning@+2{{OpenACC clause 'worker' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}} #pragma acc loop worker - // expected-error@+2{{expected expression}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+1{{expected expression}} #pragma acc loop worker() - // expected-error@+3{{invalid tag 'invalid' on 'worker' clause}} - // expected-error@+2{{expected expression}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{invalid tag 'invalid' on 'worker' clause}} + // expected-error@+1{{expected expression}} #pragma acc loop worker(invalid:) - // expected-error@+3{{invalid tag 'invalid' on 'worker' clause}} - // expected-warning@+2{{OpenACC clause 'worker' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{invalid tag 'invalid' on 'worker' clause}} + // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}} #pragma acc loop worker(invalid:5) - // expected-error@+2{{expected expression}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+1{{expected expression}} #pragma acc loop worker(num:) - // expected-error@+3{{invalid tag 'length' on 'worker' clause}} - // expected-error@+2{{expected expression}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{invalid tag 'length' on 'worker' clause}} + // expected-error@+1{{expected expression}} #pragma acc loop worker(length:) - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop worker(5, 4) - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop worker(num:6,4) - // expected-error@+4{{invalid tag 'length' on 'worker' clause}} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+3{{invalid tag 'length' on 'worker' clause}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop worker(length:6,4) - // expected-warning@+2{{OpenACC clause 'worker' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}} #pragma acc loop worker(5) - // expected-error@+3{{invalid tag 'length' on 'worker' clause}} - // expected-warning@+2{{OpenACC clause 'worker' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{invalid tag 'length' on 'worker' clause}} + // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}} #pragma acc loop worker(length:5) - // expected-warning@+2{{OpenACC clause 'worker' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}} #pragma acc loop worker(num:5) - // expected-warning@+2{{OpenACC clause 'worker' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}} #pragma acc loop worker(returns_int()) - // expected-error@+3{{invalid tag 'length' on 'worker' clause}} - // expected-warning@+2{{OpenACC clause 'worker' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + for(;;); + // expected-error@+2{{invalid tag 'length' on 'worker' clause}} + // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}} #pragma acc loop worker(length:returns_int()) + for(;;); } void device_type() { @@ -1236,238 +1228,196 @@ void AsyncArgument() { void Tile() { int* Foo; - // expected-error@+2{{expected '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+1{{expected '('}} #pragma acc loop tile for(;;){} - // expected-error@+4{{expected expression}} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop tile( for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'tile' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}} #pragma acc loop tile() for(;;){} - // expected-error@+4{{expected expression}} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop tile(, for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'tile' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}} #pragma acc loop tile(,) for(;;){} - // expected-error@+3{{use of undeclared identifier 'invalid'}} - // expected-warning@+2{{OpenACC clause 'tile' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{use of undeclared identifier 'invalid'}} + // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}} #pragma acc loop tile(returns_int(), *, invalid, *) for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'tile' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}} #pragma acc loop tile(returns_int() *, Foo, *) for(;;){} - // expected-error@+3{{indirection requires pointer operand ('int' invalid)}} - // expected-warning@+2{{OpenACC clause 'tile' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{indirection requires pointer operand ('int' invalid)}} + // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}} #pragma acc loop tile(* returns_int() , *) for(;;){} - // expected-warning@+2{{OpenACC clause 'tile' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}} #pragma acc loop tile(*) for(;;){} - // expected-warning@+2{{OpenACC clause 'tile' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}} #pragma acc loop tile(*Foo, *Foo) for(;;){} - // expected-warning@+2{{OpenACC clause 'tile' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}} #pragma acc loop tile(5) for(;;){} - // expected-warning@+2{{OpenACC clause 'tile' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}} #pragma acc loop tile(*, 5) for(;;){} - // expected-warning@+2{{OpenACC clause 'tile' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}} #pragma acc loop tile(5, *) for(;;){} - // expected-warning@+2{{OpenACC clause 'tile' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}} #pragma acc loop tile(5, *, 3, *) for(;;){} } void Gang() { - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang for(;;){} - // expected-error@+4{{expected expression}} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop gang( for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang() for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(5, *) for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(*) for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(5, num:*) for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(num:5, *) for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(num:5, num:*) for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(num:*) for(;;){} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(dim:5) for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(dim:5, dim:*) for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(dim:*) for(;;){} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(static:*) for(;;){} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(static:*, static:5) for(;;){} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(static:*, 5) for(;;){} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(static:45, 5) for(;;){} - // expected-error@+4{{expected expression}} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop gang(static:45, for(;;){} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop gang(static:45 for(;;){} - // expected-error@+4{{expected expression}} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop gang(static:*, for(;;){} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop gang(static:* for(;;){} - // expected-error@+4{{expected expression}} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop gang(45, for(;;){} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop gang(45 for(;;){} - // expected-error@+4{{expected expression}} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop gang(num:45, for(;;){} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop gang(num:45 for(;;){} - // expected-error@+4{{expected expression}} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop gang(dim:45, for(;;){} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc loop gang(dim:45 for(;;){} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(static:*, dim:returns_int(), 5) for(;;){} - // expected-warning@+2{{OpenACC clause 'gang' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}} #pragma acc loop gang(num: 32, static:*, dim:returns_int(), 5) for(;;){} diff --git a/clang/test/ParserOpenACC/parse-clauses.cpp b/clang/test/ParserOpenACC/parse-clauses.cpp index 702eb75ca890..b7e252e892be 100644 --- a/clang/test/ParserOpenACC/parse-clauses.cpp +++ b/clang/test/ParserOpenACC/parse-clauses.cpp @@ -2,13 +2,11 @@ template void templ() { - // expected-warning@+2{{OpenACC clause 'collapse' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented, clause ignored}} #pragma acc loop collapse(I) for(;;){} - // expected-warning@+2{{OpenACC clause 'collapse' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented, clause ignored}} #pragma acc loop collapse(T::value) for(;;){} diff --git a/clang/test/ParserOpenACC/parse-constructs.c b/clang/test/ParserOpenACC/parse-constructs.c index ecedfd9e9e6d..ea75360cc135 100644 --- a/clang/test/ParserOpenACC/parse-constructs.c +++ b/clang/test/ParserOpenACC/parse-constructs.c @@ -82,8 +82,7 @@ void func() { // expected-warning@+1{{OpenACC construct 'host_data' not yet implemented, pragma ignored}} #pragma acc host_data clause list for(;;){} - // expected-error@+2{{invalid OpenACC clause 'clause'}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented, pragma ignored}} + // expected-error@+1{{invalid OpenACC clause 'clause'}} #pragma acc loop clause list for(;;){} // expected-error@+1{{invalid OpenACC clause 'invalid'}} diff --git a/clang/test/SemaOpenACC/compute-construct-async-clause.c b/clang/test/SemaOpenACC/compute-construct-async-clause.c index 999db74ffbb8..fe41c5d0897a 100644 --- a/clang/test/SemaOpenACC/compute-construct-async-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-async-clause.c @@ -39,8 +39,7 @@ void Test() { #pragma acc kernels async(SomeE) while(1); - // expected-error@+2{{OpenACC 'async' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'async' clause is not valid on 'loop' directive}} #pragma acc loop async(1) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-attach-clause.c b/clang/test/SemaOpenACC/compute-construct-attach-clause.c index 769662027181..1d204094de12 100644 --- a/clang/test/SemaOpenACC/compute-construct-attach-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-attach-clause.c @@ -59,8 +59,7 @@ void uses() { #pragma acc parallel attach(s.PtrMem) while (1); - // expected-error@+2{{OpenACC 'attach' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'attach' clause is not valid on 'loop' directive}} #pragma acc loop attach(LocalInt) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-copy-clause.c b/clang/test/SemaOpenACC/compute-construct-copy-clause.c index 7adf0e18fa04..284813f21352 100644 --- a/clang/test/SemaOpenACC/compute-construct-copy-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-copy-clause.c @@ -60,16 +60,13 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo #pragma acc parallel copy((float)ArrayParam[2]) while(1); - // expected-error@+2{{OpenACC 'copy' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'copy' clause is not valid on 'loop' directive}} #pragma acc loop copy(LocalInt) for(;;); - // expected-error@+2{{OpenACC 'pcopy' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'pcopy' clause is not valid on 'loop' directive}} #pragma acc loop pcopy(LocalInt) for(;;); - // expected-error@+2{{OpenACC 'present_or_copy' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'present_or_copy' clause is not valid on 'loop' directive}} #pragma acc loop present_or_copy(LocalInt) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-copyin-clause.c b/clang/test/SemaOpenACC/compute-construct-copyin-clause.c index d55735775656..d4dda1e16737 100644 --- a/clang/test/SemaOpenACC/compute-construct-copyin-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-copyin-clause.c @@ -66,16 +66,13 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo #pragma acc parallel copyin(invalid:(float)ArrayParam[2]) while(1); - // expected-error@+2{{OpenACC 'copyin' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'copyin' clause is not valid on 'loop' directive}} #pragma acc loop copyin(LocalInt) for(;;); - // expected-error@+2{{OpenACC 'pcopyin' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'pcopyin' clause is not valid on 'loop' directive}} #pragma acc loop pcopyin(LocalInt) for(;;); - // expected-error@+2{{OpenACC 'present_or_copyin' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'present_or_copyin' clause is not valid on 'loop' directive}} #pragma acc loop present_or_copyin(LocalInt) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-copyout-clause.c b/clang/test/SemaOpenACC/compute-construct-copyout-clause.c index 432823b6746a..5692ab0f5660 100644 --- a/clang/test/SemaOpenACC/compute-construct-copyout-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-copyout-clause.c @@ -66,16 +66,13 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo #pragma acc parallel copyout(invalid:(float)ArrayParam[2]) while(1); - // expected-error@+2{{OpenACC 'copyout' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'copyout' clause is not valid on 'loop' directive}} #pragma acc loop copyout(LocalInt) for(;;); - // expected-error@+2{{OpenACC 'pcopyout' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'pcopyout' clause is not valid on 'loop' directive}} #pragma acc loop pcopyout(LocalInt) for(;;); - // expected-error@+2{{OpenACC 'present_or_copyout' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'present_or_copyout' clause is not valid on 'loop' directive}} #pragma acc loop present_or_copyout(LocalInt) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-create-clause.c b/clang/test/SemaOpenACC/compute-construct-create-clause.c index 319025c9628c..6ef9551d759e 100644 --- a/clang/test/SemaOpenACC/compute-construct-create-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-create-clause.c @@ -67,16 +67,13 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo #pragma acc parallel create(invalid:(float)ArrayParam[2]) while(1); - // expected-error@+2{{OpenACC 'create' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'create' clause is not valid on 'loop' directive}} #pragma acc loop create(LocalInt) for(;;); - // expected-error@+2{{OpenACC 'pcreate' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'pcreate' clause is not valid on 'loop' directive}} #pragma acc loop pcreate(LocalInt) for(;;); - // expected-error@+2{{OpenACC 'present_or_create' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'present_or_create' clause is not valid on 'loop' directive}} #pragma acc loop present_or_create(LocalInt) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-default-clause.c b/clang/test/SemaOpenACC/compute-construct-default-clause.c index bcafb02cb4df..93e8f7c2a6b1 100644 --- a/clang/test/SemaOpenACC/compute-construct-default-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-default-clause.c @@ -43,18 +43,16 @@ void SingleOnly() { #pragma acc data default(none) while(0); - // expected-warning@+2{{OpenACC construct 'loop' not yet implemented}} // expected-error@+1{{OpenACC 'default' clause is not valid on 'loop' directive}} #pragma acc loop default(none) - while(0); + for(;;); // expected-warning@+2{{OpenACC construct 'wait' not yet implemented}} // expected-error@+1{{OpenACC 'default' clause is not valid on 'wait' directive}} #pragma acc wait default(none) while(0); - // expected-error@+2{{OpenACC 'default' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'default' clause is not valid on 'loop' directive}} #pragma acc loop default(present) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-deviceptr-clause.c b/clang/test/SemaOpenACC/compute-construct-deviceptr-clause.c index 8ec911f6dbf1..44c4cc4e5ec2 100644 --- a/clang/test/SemaOpenACC/compute-construct-deviceptr-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-deviceptr-clause.c @@ -59,8 +59,7 @@ void uses() { #pragma acc parallel deviceptr(s.PtrMem) while (1); - // expected-error@+2{{OpenACC 'deviceptr' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'deviceptr' clause is not valid on 'loop' directive}} #pragma acc loop deviceptr(LocalInt) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-firstprivate-clause.c b/clang/test/SemaOpenACC/compute-construct-firstprivate-clause.c index 14f5af60cc85..0c26a0b4c9b9 100644 --- a/clang/test/SemaOpenACC/compute-construct-firstprivate-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-firstprivate-clause.c @@ -53,8 +53,7 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo #pragma acc parallel firstprivate((float)ArrayParam[2]) while(1); - // expected-error@+2{{OpenACC 'firstprivate' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'firstprivate' clause is not valid on 'loop' directive}} #pragma acc loop firstprivate(LocalInt) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-if-clause.c b/clang/test/SemaOpenACC/compute-construct-if-clause.c index 21e7ce413e90..4629b1b2c2bd 100644 --- a/clang/test/SemaOpenACC/compute-construct-if-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-if-clause.c @@ -60,8 +60,7 @@ void BoolExpr(int *I, float *F) { #pragma acc kernels loop if (*I < *F) while(0); - // expected-error@+2{{OpenACC 'if' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'if' clause is not valid on 'loop' directive}} #pragma acc loop if(I) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-no_create-clause.c b/clang/test/SemaOpenACC/compute-construct-no_create-clause.c index 5afd64446214..6db7d0cca8c3 100644 --- a/clang/test/SemaOpenACC/compute-construct-no_create-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-no_create-clause.c @@ -52,8 +52,7 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo #pragma acc parallel no_create((float)ArrayParam[2]) while(1); - // expected-error@+2{{OpenACC 'no_create' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'no_create' clause is not valid on 'loop' directive}} #pragma acc loop no_create(LocalInt) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.c b/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.c index 9c2a5a781059..0a86dee4da04 100644 --- a/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.c @@ -52,8 +52,7 @@ void Test() { #pragma acc parallel num_gangs(getS(), 1, getS(), 1) while(1); - // expected-error@+2{{OpenACC 'num_gangs' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'num_gangs' clause is not valid on 'loop' directive}} #pragma acc loop num_gangs(1) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-num_workers-clause.c b/clang/test/SemaOpenACC/compute-construct-num_workers-clause.c index a84bd3699536..808609cf2a0f 100644 --- a/clang/test/SemaOpenACC/compute-construct-num_workers-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-num_workers-clause.c @@ -31,8 +31,7 @@ void Test() { #pragma acc kernels num_workers(SomeE) while(1); - // expected-error@+2{{OpenACC 'num_workers' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'num_workers' clause is not valid on 'loop' directive}} #pragma acc loop num_workers(1) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-present-clause.c b/clang/test/SemaOpenACC/compute-construct-present-clause.c index 5ace750da7ef..eea2c77657c8 100644 --- a/clang/test/SemaOpenACC/compute-construct-present-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-present-clause.c @@ -52,8 +52,7 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo #pragma acc parallel present((float)ArrayParam[2]) while(1); - // expected-error@+2{{OpenACC 'present' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'present' clause is not valid on 'loop' directive}} #pragma acc loop present(LocalInt) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-self-clause.c b/clang/test/SemaOpenACC/compute-construct-self-clause.c index 634a2d8857b7..c79e7e5d3db6 100644 --- a/clang/test/SemaOpenACC/compute-construct-self-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-self-clause.c @@ -80,8 +80,7 @@ void WarnMaybeNotUsed(int val1, int val2) { #pragma acc parallel if(invalid) self(val1) while(0); - // expected-error@+2{{OpenACC 'self' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'self' clause is not valid on 'loop' directive}} #pragma acc loop self for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-vector_length-clause.c b/clang/test/SemaOpenACC/compute-construct-vector_length-clause.c index 83055f81fbb2..eda2d5e251b2 100644 --- a/clang/test/SemaOpenACC/compute-construct-vector_length-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-vector_length-clause.c @@ -31,8 +31,7 @@ void Test() { #pragma acc kernels vector_length(SomeE) while(1); - // expected-error@+2{{OpenACC 'vector_length' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'vector_length' clause is not valid on 'loop' directive}} #pragma acc loop vector_length(1) for(;;); } diff --git a/clang/test/SemaOpenACC/compute-construct-wait-clause.c b/clang/test/SemaOpenACC/compute-construct-wait-clause.c index 0878288ca4a2..0d0ab52c31dc 100644 --- a/clang/test/SemaOpenACC/compute-construct-wait-clause.c +++ b/clang/test/SemaOpenACC/compute-construct-wait-clause.c @@ -36,8 +36,7 @@ void uses() { #pragma acc parallel wait(devnum:arr : queues: arr, NC, 5) while(1); - // expected-error@+2{{OpenACC 'wait' clause is not valid on 'loop' directive}} - // expected-warning@+1{{OpenACC construct 'loop' not yet implemented}} + // expected-error@+1{{OpenACC 'wait' clause is not valid on 'loop' directive}} #pragma acc loop wait for(;;); } diff --git a/clang/test/SemaOpenACC/loop-ast.cpp b/clang/test/SemaOpenACC/loop-ast.cpp new file mode 100644 index 000000000000..292044f94267 --- /dev/null +++ b/clang/test/SemaOpenACC/loop-ast.cpp @@ -0,0 +1,182 @@ + +// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s + +// Test this with PCH. +// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s +// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s + +#ifndef PCH_HELPER +#define PCH_HELPER + +void NormalFunc() { + // CHECK-LABEL: NormalFunc + // CHECK-NEXT: CompoundStmt + +#pragma acc loop + for(;;); + // CHECK-NEXT: OpenACCLoopConstruct{{.*}} + // CHECK-NEXT: ForStmt + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: NullStmt + + int array[5]; + // CHECK-NEXT: DeclStmt + // CHECK-NEXT: VarDecl +#pragma acc loop + for(auto x : array){} + // CHECK-NEXT: OpenACCLoopConstruct{{.*}} + // CHECK-NEXT: CXXForRangeStmt + // CHECK: CompoundStmt + +#pragma acc parallel + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}parallel + // CHECK-NEXT: CompoundStmt + { +#pragma acc parallel + // CHECK-NEXT: OpenACCComputeConstruct [[PAR_ADDR:[0-9a-fx]+]] {{.*}}parallel + // CHECK-NEXT: CompoundStmt + { +#pragma acc loop + for(;;); + // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: [[PAR_ADDR]] + // CHECK-NEXT: ForStmt + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: NullStmt + } + } +} + +template +void TemplFunc() { + // CHECK-LABEL: FunctionTemplateDecl {{.*}}TemplFunc + // CHECK-NEXT: TemplateTypeParmDecl + // CHECK-NEXT: FunctionDecl{{.*}}TemplFunc + // CHECK-NEXT: CompoundStmt + +#pragma acc loop + // CHECK-NEXT: OpenACCLoopConstruct{{.*}} + for(typename T::type t = 0; t < 5;++t) { + // CHECK-NEXT: ForStmt + // CHECK-NEXT: DeclStmt + // CHECK-NEXT: VarDecl{{.*}} referenced t 'typename T::type' + // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 0 + // CHECK-NEXT: <<> + // CHECK-NEXT: BinaryOperator{{.*}} '' '<' + // CHECK-NEXT: DeclRefExpr {{.*}} 'typename T::type' lvalue Var + // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 5 + // CHECK-NEXT: UnaryOperator{{.*}} '' lvalue prefix '++' + // CHECK-NEXT: DeclRefExpr {{.*}} 'typename T::type' lvalue Var + // CHECK-NEXT: CompoundStmt + typename T::type I; + // CHECK-NEXT: DeclStmt + // CHECK-NEXT: VarDecl{{.*}} I 'typename T::type' + + } + +#pragma acc parallel + { + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}parallel + // CHECK-NEXT: CompoundStmt +#pragma acc parallel + { + // CHECK-NEXT: OpenACCComputeConstruct [[PAR_ADDR_UNINST:[0-9a-fx]+]] {{.*}}parallel + // CHECK-NEXT: CompoundStmt +#pragma acc loop + // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: [[PAR_ADDR_UNINST]] + // CHECK-NEXT: ForStmt + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: NullStmt + for(;;); + +#pragma acc loop + // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: [[PAR_ADDR_UNINST]] + // CHECK-NEXT: ForStmt + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: NullStmt + for(;;); + } + } + + typename T::type array[5]; + // CHECK-NEXT: DeclStmt + // CHECK-NEXT: VarDecl + +#pragma acc loop + for(auto x : array){} + // CHECK-NEXT: OpenACCLoopConstruct{{.*}} + // CHECK-NEXT: CXXForRangeStmt + // CHECK: CompoundStmt + + // Instantiation: + // CHECK-NEXT: FunctionDecl{{.*}} TemplFunc 'void ()' implicit_instantiation + // CHECK-NEXT: TemplateArgument type 'S' + // CHECK-NEXT: RecordType{{.*}} 'S' + // CHECK-NEXT: CXXRecord{{.*}} 'S' + // CHECK-NEXT: CompoundStmt + + // CHECK-NEXT: OpenACCLoopConstruct{{.*}} + // CHECK-NEXT: ForStmt + // CHECK-NEXT: DeclStmt + // CHECK-NEXT: VarDecl{{.*}} used t 'typename S::type':'int' + // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 0 + // CHECK-NEXT: <<> + // CHECK-NEXT: BinaryOperator{{.*}} 'bool' '<' + // CHECK-NEXT: ImplicitCastExpr{{.*}} 'typename S::type':'int' + // CHECK-NEXT: DeclRefExpr {{.*}} 'typename S::type':'int' lvalue Var + // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 5 + // CHECK-NEXT: UnaryOperator{{.*}} 'typename S::type':'int' lvalue prefix '++' + // CHECK-NEXT: DeclRefExpr {{.*}} 'typename S::type':'int' lvalue Var + // CHECK-NEXT: CompoundStmt + // CHECK-NEXT: DeclStmt + // CHECK-NEXT: VarDecl{{.*}} I 'typename S::type':'int' + + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}parallel + // CHECK-NEXT: CompoundStmt + // + // CHECK-NEXT: OpenACCComputeConstruct [[PAR_ADDR_INST:[0-9a-fx]+]] {{.*}}parallel + // CHECK-NEXT: CompoundStmt + + // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: [[PAR_ADDR_INST]] + // CHECK-NEXT: ForStmt + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: NullStmt + + // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: [[PAR_ADDR_INST]] + // CHECK-NEXT: ForStmt + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: <<>> + // CHECK-NEXT: NullStmt + + // CHECK-NEXT: DeclStmt + // CHECK-NEXT: VarDecl + // CHECK-NEXT: OpenACCLoopConstruct{{.*}} + // CHECK-NEXT: CXXForRangeStmt + // CHECK: CompoundStmt +} + +struct S { + using type = int; +}; + +void use() { + TemplFunc(); +} +#endif + diff --git a/clang/test/SemaOpenACC/loop-loc-and-stmt.c b/clang/test/SemaOpenACC/loop-loc-and-stmt.c new file mode 100644 index 000000000000..36c6743f9843 --- /dev/null +++ b/clang/test/SemaOpenACC/loop-loc-and-stmt.c @@ -0,0 +1,38 @@ +// RUN: %clang_cc1 %s -verify -fopenacc + +// expected-error@+1{{OpenACC construct 'loop' cannot be used here; it can only be used in a statement context}} +#pragma acc loop + +// expected-error@+1{{OpenACC construct 'loop' cannot be used here; it can only be used in a statement context}} +#pragma acc loop +int foo; + +struct S { +// expected-error@+1{{OpenACC construct 'loop' cannot be used here; it can only be used in a statement context}} +#pragma acc loop + int i; +}; + +void func() { + // expected-error@+2{{expected expression}} +#pragma acc loop + int foo; + + // expected-error@+3{{OpenACC 'loop' construct can only be applied to a 'for' loop}} + // expected-note@+1{{'loop' construct is here}} +#pragma acc loop + while(0); + + // expected-error@+3{{OpenACC 'loop' construct can only be applied to a 'for' loop}} + // expected-note@+1{{'loop' construct is here}} +#pragma acc loop + do{}while(0); + + // expected-error@+3{{OpenACC 'loop' construct can only be applied to a 'for' loop}} + // expected-note@+1{{'loop' construct is here}} +#pragma acc loop + {} + +#pragma acc loop + for(;;); +} diff --git a/clang/test/SemaOpenACC/loop-loc-and-stmt.cpp b/clang/test/SemaOpenACC/loop-loc-and-stmt.cpp new file mode 100644 index 000000000000..5d50145b7c88 --- /dev/null +++ b/clang/test/SemaOpenACC/loop-loc-and-stmt.cpp @@ -0,0 +1,80 @@ +// RUN: %clang_cc1 %s -verify -fopenacc +// +// expected-error@+1{{OpenACC construct 'loop' cannot be used here; it can only be used in a statement context}} +#pragma acc loop + +// expected-error@+1{{OpenACC construct 'loop' cannot be used here; it can only be used in a statement context}} +#pragma acc loop +int foo; + +struct S { +// expected-error@+1{{OpenACC construct 'loop' cannot be used here; it can only be used in a statement context}} +#pragma acc loop + int i; + + void mem_func() { + // expected-error@+3{{OpenACC 'loop' construct can only be applied to a 'for' loop}} + // expected-note@+1{{'loop' construct is here}} +#pragma acc loop + int foo; + + // expected-error@+3{{OpenACC 'loop' construct can only be applied to a 'for' loop}} + // expected-note@+1{{'loop' construct is here}} +#pragma acc loop + while(0); + + // expected-error@+3{{OpenACC 'loop' construct can only be applied to a 'for' loop}} + // expected-note@+1{{'loop' construct is here}} +#pragma acc loop + do{}while(0); + + // expected-error@+3{{OpenACC 'loop' construct can only be applied to a 'for' loop}} + // expected-note@+1{{'loop' construct is here}} +#pragma acc loop + {} + +#pragma acc loop + for(;;); + + int array[5]; + +#pragma acc loop + for(auto X : array){} +} +}; + +template +void templ_func() { + // expected-error@+3{{OpenACC 'loop' construct can only be applied to a 'for' loop}} + // expected-note@+1{{'loop' construct is here}} +#pragma acc loop + int foo; + + // expected-error@+3{{OpenACC 'loop' construct can only be applied to a 'for' loop}} + // expected-note@+1{{'loop' construct is here}} +#pragma acc loop + while(T{}); + + // expected-error@+3{{OpenACC 'loop' construct can only be applied to a 'for' loop}} + // expected-note@+1{{'loop' construct is here}} +#pragma acc loop + do{}while(0); + + // expected-error@+3{{OpenACC 'loop' construct can only be applied to a 'for' loop}} + // expected-note@+1{{'loop' construct is here}} +#pragma acc loop + {} + +#pragma acc loop + for(T i;;); + + T array[5]; + +#pragma acc loop + for(auto X : array){} +} + +void use() { + templ_func(); +} + diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp index 49ed60d990ca..916e941cfbde 100644 --- a/clang/tools/libclang/CIndex.cpp +++ b/clang/tools/libclang/CIndex.cpp @@ -2170,6 +2170,7 @@ public: void VisitRequiresExpr(const RequiresExpr *E); void VisitCXXParenListInitExpr(const CXXParenListInitExpr *E); void VisitOpenACCComputeConstruct(const OpenACCComputeConstruct *D); + void VisitOpenACCLoopConstruct(const OpenACCLoopConstruct *D); void VisitOMPExecutableDirective(const OMPExecutableDirective *D); void VisitOMPLoopBasedDirective(const OMPLoopBasedDirective *D); void VisitOMPLoopDirective(const OMPLoopDirective *D); @@ -3496,6 +3497,12 @@ void EnqueueVisitor::VisitOpenACCComputeConstruct( EnqueueChildren(Clause); } +void EnqueueVisitor::VisitOpenACCLoopConstruct(const OpenACCLoopConstruct *C) { + EnqueueChildren(C); + for (auto *Clause : C->clauses()) + EnqueueChildren(Clause); +} + void EnqueueVisitor::VisitAnnotateAttr(const AnnotateAttr *A) { EnqueueChildren(A); } @@ -6234,6 +6241,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) { return cxstring::createRef("ConceptDecl"); case CXCursor_OpenACCComputeConstruct: return cxstring::createRef("OpenACCComputeConstruct"); + case CXCursor_OpenACCLoopConstruct: + return cxstring::createRef("OpenACCLoopConstruct"); } llvm_unreachable("Unhandled CXCursorKind"); diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp index 9325a16d2a84..38002052227c 100644 --- a/clang/tools/libclang/CXCursor.cpp +++ b/clang/tools/libclang/CXCursor.cpp @@ -873,6 +873,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent, case Stmt::OpenACCComputeConstructClass: K = CXCursor_OpenACCComputeConstruct; break; + case Stmt::OpenACCLoopConstructClass: + K = CXCursor_OpenACCLoopConstruct; + break; case Stmt::OMPTargetParallelGenericLoopDirectiveClass: K = CXCursor_OMPTargetParallelGenericLoopDirective; break; -- GitLab From 3387e558449e1748a7d4b10f2d9049647c9acc56 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Wed, 5 Jun 2024 14:28:40 +0200 Subject: [PATCH 025/675] [InstCombine] Use SimplifyQuery in isKnownSign() This enabled the use of DomConditionCache. As such, remove the explicit isImpliedByDomCondition() call. This is probably not entirely NFC because these APIs don't support exactly the same cases. --- .../InstCombine/InstCombineCalls.cpp | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index b6f339da31f7..0632f3cfc6dd 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -1041,10 +1041,8 @@ Instruction *InstCombinerImpl::foldIntrinsicIsFPClass(IntrinsicInst &II) { return nullptr; } -static std::optional getKnownSign(Value *Op, Instruction *CxtI, - const DataLayout &DL, AssumptionCache *AC, - DominatorTree *DT) { - KnownBits Known = computeKnownBits(Op, DL, 0, AC, CxtI, DT); +static std::optional getKnownSign(Value *Op, const SimplifyQuery &SQ) { + KnownBits Known = computeKnownBits(Op, /*Depth=*/0, SQ); if (Known.isNonNegative()) return false; if (Known.isNegative()) @@ -1052,34 +1050,30 @@ static std::optional getKnownSign(Value *Op, Instruction *CxtI, Value *X, *Y; if (match(Op, m_NSWSub(m_Value(X), m_Value(Y)))) - return isImpliedByDomCondition(ICmpInst::ICMP_SLT, X, Y, CxtI, DL); + return isImpliedByDomCondition(ICmpInst::ICMP_SLT, X, Y, SQ.CxtI, SQ.DL); - return isImpliedByDomCondition( - ICmpInst::ICMP_SLT, Op, Constant::getNullValue(Op->getType()), CxtI, DL); + return std::nullopt; } -static std::optional getKnownSignOrZero(Value *Op, Instruction *CxtI, - const DataLayout &DL, - AssumptionCache *AC, - DominatorTree *DT) { - if (std::optional Sign = getKnownSign(Op, CxtI, DL, AC, DT)) +static std::optional getKnownSignOrZero(Value *Op, + const SimplifyQuery &SQ) { + if (std::optional Sign = getKnownSign(Op, SQ)) return Sign; Value *X, *Y; if (match(Op, m_NSWSub(m_Value(X), m_Value(Y)))) - return isImpliedByDomCondition(ICmpInst::ICMP_SLE, X, Y, CxtI, DL); + return isImpliedByDomCondition(ICmpInst::ICMP_SLE, X, Y, SQ.CxtI, SQ.DL); return std::nullopt; } /// Return true if two values \p Op0 and \p Op1 are known to have the same sign. -static bool signBitMustBeTheSame(Value *Op0, Value *Op1, Instruction *CxtI, - const DataLayout &DL, AssumptionCache *AC, - DominatorTree *DT) { - std::optional Known1 = getKnownSign(Op1, CxtI, DL, AC, DT); +static bool signBitMustBeTheSame(Value *Op0, Value *Op1, + const SimplifyQuery &SQ) { + std::optional Known1 = getKnownSign(Op1, SQ); if (!Known1) return false; - std::optional Known0 = getKnownSign(Op0, CxtI, DL, AC, DT); + std::optional Known0 = getKnownSign(Op0, SQ); if (!Known0) return false; return *Known0 == *Known1; @@ -1628,7 +1622,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { } if (std::optional Known = - getKnownSignOrZero(IIOperand, II, DL, &AC, &DT)) { + getKnownSignOrZero(IIOperand, SQ.getWithInstruction(II))) { // abs(x) -> x if x >= 0 (include abs(x-y) --> x - y where x >= y) // abs(x) -> x if x > 0 (include abs(x-y) --> x - y where x > y) if (!*Known) @@ -1753,7 +1747,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { bool UseAndN = IID == Intrinsic::smin || IID == Intrinsic::umin; if (IID == Intrinsic::smax || IID == Intrinsic::smin) { - auto KnownSign = getKnownSign(X, II, DL, &AC, &DT); + auto KnownSign = getKnownSign(X, SQ.getWithInstruction(II)); if (KnownSign == std::nullopt) { UseOr = false; UseAndN = false; @@ -2614,7 +2608,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { FastMathFlags InnerFlags = cast(Src)->getFastMathFlags(); if ((FMF.allowReassoc() && InnerFlags.allowReassoc()) || - signBitMustBeTheSame(Exp, InnerExp, II, DL, &AC, &DT)) { + signBitMustBeTheSame(Exp, InnerExp, SQ.getWithInstruction(II))) { // TODO: Add nsw/nuw probably safe if integer type exceeds exponent // width. Value *NewExp = Builder.CreateAdd(InnerExp, Exp); -- GitLab From 79e09b1555a51cc8d02d0225ed6d81a21fa09eca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hana=20Dus=C3=ADkov=C3=A1?= Date: Wed, 5 Jun 2024 15:43:05 +0200 Subject: [PATCH 026/675] [llvm-cov] [NFC] don't test pseudo-selectors in CSS fixes test for .css file generated by llvm-cov from recent PR https://github.com/llvm/llvm-project/pull/93080 --- llvm/test/tools/llvm-cov/style.test | 6 ------ 1 file changed, 6 deletions(-) diff --git a/llvm/test/tools/llvm-cov/style.test b/llvm/test/tools/llvm-cov/style.test index 735050e1de30..d213d2492c61 100644 --- a/llvm/test/tools/llvm-cov/style.test +++ b/llvm/test/tools/llvm-cov/style.test @@ -23,19 +23,13 @@ STYLE-DAG: .light-row-bold STYLE-DAG: .column-entry STYLE-DAG: .column-entry-bold STYLE-DAG: .column-entry-yellow -STYLE-DAG: .column-entry-yellow:hover STYLE-DAG: .column-entry-red -STYLE-DAG: .column-entry-red:hover STYLE-DAG: .column-entry-green -STYLE-DAG: .column-entry-green:hover STYLE-DAG: .covered-line STYLE-DAG: .uncovered-line STYLE-DAG: .tooltip STYLE-DAG: .tooltip span.tooltip-content STYLE-DAG: th, td -STYLE-DAG: td:first-child -STYLE-DAG: td:last-child -STYLE-DAG: tr:hover TOPLEVEL-NOT: