diff options
217 files changed, 7437 insertions, 2641 deletions
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h index 7b10b2d..118d579 100644 --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -2142,8 +2142,9 @@ public: } /// Detects whether \p Address is inside a data region in this function - /// (constant islands). - bool isInConstantIsland(uint64_t Address) const { + /// (constant islands), and optionally return the island size starting + /// from the given \p Address. + bool isInConstantIsland(uint64_t Address, uint64_t *Size = nullptr) const { if (!Islands) return false; @@ -2161,10 +2162,15 @@ public: DataIter = std::prev(DataIter); auto CodeIter = Islands->CodeOffsets.upper_bound(Offset); - if (CodeIter == Islands->CodeOffsets.begin()) + if (CodeIter == Islands->CodeOffsets.begin() || + *std::prev(CodeIter) <= *DataIter) { + if (Size) + *Size = (CodeIter == Islands->CodeOffsets.end() ? getMaxSize() + : *CodeIter) - + Offset; return true; - - return *std::prev(CodeIter) <= *DataIter; + } + return false; } uint16_t getConstantIslandAlignment() const; diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp index 7dded4c..c33540a 100644 --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -77,6 +77,11 @@ cl::opt<std::string> CompDirOverride( "location, which is used with DW_AT_dwo_name to construct a path " "to *.dwo files."), cl::Hidden, cl::init(""), cl::cat(BoltCategory)); + +static cl::opt<bool> + FailOnInvalidPadding("fail-on-invalid-padding", cl::Hidden, cl::init(false), + cl::desc("treat invalid code padding as error"), + cl::ZeroOrMore, cl::cat(BoltCategory)); } // namespace opts namespace llvm { @@ -942,8 +947,7 @@ std::string BinaryContext::generateJumpTableName(const BinaryFunction &BF, } bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) { - // FIXME: aarch64 support is missing. - if (!isX86()) + if (!isX86() && !isAArch64()) return true; if (BF.getSize() == BF.getMaxSize()) @@ -973,14 +977,26 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) { return Offset - StartOffset; }; - // Skip a sequence of zero bytes. + // Skip a sequence of zero bytes. For AArch64 we only skip 4 bytes of zeros + // in case the following zeros belong to constant island or veneer. auto skipZeros = [&]() { const uint64_t StartOffset = Offset; - for (; Offset < BF.getMaxSize(); ++Offset) - if ((*FunctionData)[Offset] != 0) + uint64_t CurrentOffset = Offset; + for (; CurrentOffset < BF.getMaxSize() && + (!isAArch64() || CurrentOffset < StartOffset + 4); + ++CurrentOffset) + if ((*FunctionData)[CurrentOffset] != 0) break; - return Offset - StartOffset; + uint64_t NumZeros = CurrentOffset - StartOffset; + if (isAArch64()) + NumZeros &= ~((uint64_t)0x3); + + if (NumZeros == 0) + return false; + Offset += NumZeros; + InstrAddress += NumZeros; + return true; }; // Accept the whole padding area filled with breakpoints. @@ -993,6 +1009,8 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) { // Some functions have a jump to the next function or to the padding area // inserted after the body. auto isSkipJump = [&](const MCInst &Instr) { + if (!isX86()) + return false; uint64_t TargetAddress = 0; if (MIB->isUnconditionalBranch(Instr) && MIB->evaluateBranch(Instr, InstrAddress, InstrSize, TargetAddress)) { @@ -1004,34 +1022,73 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) { return false; }; + // For veneers that are not already covered by binary functions, only those + // that handleAArch64Veneer() can recognize are checked here. + auto skipAArch64Veneer = [&]() { + if (!isAArch64() || Offset >= BF.getMaxSize()) + return false; + BinaryFunction *BFVeneer = getBinaryFunctionContainingAddress(InstrAddress); + if (BFVeneer) { + // A binary function may have been created to point to this veneer. + Offset += BFVeneer->getSize(); + assert(Offset <= BF.getMaxSize() && + "AArch64 veneeer goes past the max size of function"); + InstrAddress += BFVeneer->getSize(); + return true; + } + const uint64_t AArch64VeneerSize = 12; + if (Offset + AArch64VeneerSize <= BF.getMaxSize() && + handleAArch64Veneer(InstrAddress, /*MatchOnly*/ true)) { + Offset += AArch64VeneerSize; + InstrAddress += AArch64VeneerSize; + this->errs() << "BOLT-WARNING: found unmarked AArch64 veneer at 0x" + << Twine::utohexstr(BF.getAddress() + Offset) << '\n'; + return true; + } + return false; + }; + + auto skipAArch64ConstantIsland = [&]() { + if (!isAArch64() || Offset >= BF.getMaxSize()) + return false; + uint64_t Size; + if (BF.isInConstantIsland(InstrAddress, &Size)) { + Offset += Size; + InstrAddress += Size; + return true; + } + return false; + }; + // Skip over nops, jumps, and zero padding. Allow interleaving (this happens). - while (skipInstructions(isNoop) || skipInstructions(isSkipJump) || + // For AArch64 also check veneers and skip constant islands. + while (skipAArch64Veneer() || skipAArch64ConstantIsland() || + skipInstructions(isNoop) || skipInstructions(isSkipJump) || skipZeros()) ; if (Offset == BF.getMaxSize()) return true; - if (opts::Verbosity >= 1) { - this->errs() << "BOLT-WARNING: bad padding at address 0x" - << Twine::utohexstr(BF.getAddress() + BF.getSize()) - << " starting at offset " << (Offset - BF.getSize()) - << " in function " << BF << '\n' - << FunctionData->slice(BF.getSize(), - BF.getMaxSize() - BF.getSize()) - << '\n'; - } - + this->errs() << "BOLT-WARNING: bad padding at address 0x" + << Twine::utohexstr(BF.getAddress() + BF.getSize()) + << " starting at offset " << (Offset - BF.getSize()) + << " in function " << BF << '\n' + << FunctionData->slice(BF.getSize(), + BF.getMaxSize() - BF.getSize()) + << '\n'; return false; } void BinaryContext::adjustCodePadding() { + uint64_t NumInvalid = 0; for (auto &BFI : BinaryFunctions) { BinaryFunction &BF = BFI.second; if (!shouldEmit(BF)) continue; if (!hasValidCodePadding(BF)) { + NumInvalid++; if (HasRelocations) { this->errs() << "BOLT-WARNING: function " << BF << " has invalid padding. Ignoring the function\n"; @@ -1041,6 +1098,11 @@ void BinaryContext::adjustCodePadding() { } } } + if (NumInvalid && opts::FailOnInvalidPadding) { + this->errs() << "BOLT-ERROR: found " << NumInvalid + << " instance(s) of invalid code padding\n"; + exit(1); + } } MCSymbol *BinaryContext::registerNameAtAddress(StringRef Name, uint64_t Address, diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index 4dfd4ba..776f4ae 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -1427,10 +1427,9 @@ Error BinaryFunction::disassemble() { !(BC.isAArch64() && BC.handleAArch64Veneer(TargetAddress, /*MatchOnly*/ true))) { // Result of __builtin_unreachable(). - LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump past end detected at 0x" - << Twine::utohexstr(AbsoluteInstrAddr) - << " in function " << *this - << " : replacing with nop.\n"); + errs() << "BOLT-WARNING: jump past end detected at 0x" + << Twine::utohexstr(AbsoluteInstrAddr) << " in function " + << *this << " : replacing with nop.\n"; BC.MIB->createNoop(Instruction); if (IsCondBranch) { // Register branch offset for profile validation. diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp index 6954cb2..7769162 100644 --- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp +++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp @@ -157,6 +157,10 @@ public: MCPhysReg getStackPointer() const override { return AArch64::SP; } MCPhysReg getFramePointer() const override { return AArch64::FP; } + bool isBreakpoint(const MCInst &Inst) const override { + return Inst.getOpcode() == AArch64::BRK; + } + bool isPush(const MCInst &Inst) const override { return isStoreToStack(Inst); }; @@ -2153,7 +2157,7 @@ public: --I; Address -= 4; - if (I->getOpcode() != AArch64::ADRP || + if (I != Begin || I->getOpcode() != AArch64::ADRP || MCPlus::getNumPrimeOperands(*I) < 2 || !I->getOperand(0).isReg() || !I->getOperand(1).isImm() || I->getOperand(0).getReg() != AArch64::X16) return 0; diff --git a/bolt/test/AArch64/Inputs/plt-gnu-ld.yaml b/bolt/test/AArch64/Inputs/plt-gnu-ld.yaml index 1e3353a..f6dc60d 100644 --- a/bolt/test/AArch64/Inputs/plt-gnu-ld.yaml +++ b/bolt/test/AArch64/Inputs/plt-gnu-ld.yaml @@ -93,7 +93,7 @@ Sections: Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x400510 AddressAlign: 0x8 - Content: 1D0080D21E0080D2E50300AAE10340F9E2230091E603009100000090002015910300009063201B910400009084201D91E0FFFF97EBFFFF972F0000148000009000F047F9400000B4E2FFFF17C0035FD6800000B000000191810000B0210001913F0000EBC00000540100009021B443F9610000B4F00301AA00021FD6C0035FD6800000B000000191810000B021000191210000CB22FC7FD3410C818BFF0781EB21FC4193C00000540200009042B843F9620000B4F00302AA00021FD6C0035FD6FD7BBEA9FD030091F30B00F9930000B06002413980000035DEFFFF972000805260020139F30B40F9FD7BC2A8C0035FD6E4FFFF17FF8300D1FD7B01A9FD430091BFC31FB8E1230091E8DD9752A8D5BB72E80300B9E80B00B9E0130091FF0700B9880000B00900009029C11291092500F9082540F9820080D200013FD6E90340B9E80740B90801096BA00000540100001428008052A8C31FB814000014880000B00900009029411391092900F9082940F9E0230091E1031F2A820080D200013FD6E80B40B9A80000340100001428008052A8C31FB8050000140000009000E01D9194FFFF9701000014A0C35FB8FD7B41A9FF830091C0035FD6FD7BBCA9FD030091F35301A99400009094023891F55B02A995000090B5E23791940215CBF603002AF76303A9F70301AAF80302AA5DFFFF97FF0F94EB6001005494FE4393130080D2A37A73F8E20318AA73060091E10317AAE003162A60003FD69F0213EB21FFFF54F35341A9F55B42A9F76343A9FD7BC4A8C0035FD61F2003D5C0035FD6 + Content: 1D0080D21E0080D2E50300AAE10340F9E2230091E603009100000090002015910300009063201B910400009084201D91E0FFFF97EBFFFF972F0000148000009000F047F9400000B4E2FFFF17C0035FD6800000B000000191810000B0210001913F0000EBC00000540100009021B443F9610000B4F00301AA00021FD6C0035FD6800000B000000191810000B021000191210000CB22FC7FD3410C818BFF0781EB21FC4193C00000540200009042B843F9620000B4F00302AA00021FD6C0035FD6FD7BBEA9FD030091F30B00F9930000B06002413980000035DEFFFF972000805260020139F30B40F9FD7BC2A8C0035FD6E4FFFF17FF8300D1FD7B01A9FD430091BFC31FB8E1230091E8DD9752A8D5BB72E80300B9E80B00B9E0130091FF0700B9880000B00900009029C11291092500F9082540F9820080D200013FD6E90340B9E80740B90801096BA00000540100001428008052A8C31FB814000014880000B00900009029411391092900F9082940F9E0230091E1031F2A820080D200013FD6E80B40B9A80000340100001428008052A8C31FB8050000140000009000E01D9194FFFF9701000014A0C35FB8FD7B41A9FF830091C0035FD6FD7BBCA9FD030091F35301A99400009094023891F55B02A995000090B5E23791940215CBF603002AF76303A9F70301AAF80302AA60003FD6FF0F94EB6001005494FE4393130080D2A37A73F8E20318AA73060091E10317AAE003162A60003FD69F0213EB21FFFF54F35341A9F55B42A9F76343A9FD7BC4A8C0035FD61F2003D5C0035FD6 - Name: .rodata Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] @@ -435,6 +435,12 @@ Symbols: Binding: STB_GLOBAL Value: 0x400604 Size: 0xC4 + - Name: post_main + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x4006C8 + Size: 0x7C - Name: 'printf@@GLIBC_2.17' Type: STT_FUNC Binding: STB_GLOBAL diff --git a/bolt/test/AArch64/invalid-code-padding.s b/bolt/test/AArch64/invalid-code-padding.s new file mode 100644 index 0000000..4706e60 --- /dev/null +++ b/bolt/test/AArch64/invalid-code-padding.s @@ -0,0 +1,34 @@ +# RUN: %clang %cflags %s -o %t.so -Wl,-q +# RUN: not llvm-bolt %t.so -o %t.bolt --fail-on-invalid-padding \ +# RUN: 2>&1 | FileCheck %s +# CHECK: BOLT-ERROR: found 1 instance(s) of invalid code padding + + .text + .align 2 + .global foo + .type foo, %function +foo: + cmp x0, x1 + b.eq .Ltmp1 + adrp x1, jmptbl + add x1, x1, :lo12:jmptbl + ldrsw x2, [x1, x2, lsl #2] + br x2 + b .Ltmp1 +.Ltmp2: + add x0, x0, x1 + ret + .size foo, .-foo + +.Ltmp1: + add x0, x0, x1 + b .Ltmp2 + + # Dummy relocation to force relocation mode + .reloc 0, R_AARCH64_NONE + + .section .rodata, "a" + .align 2 + .global jmptbl +jmptbl: + .word .text+0x28 - . diff --git a/bolt/test/AArch64/plt-got.test b/bolt/test/AArch64/plt-got.test index be1c095..18b1ea4 100644 --- a/bolt/test/AArch64/plt-got.test +++ b/bolt/test/AArch64/plt-got.test @@ -1,7 +1,8 @@ // This test checks .plt.got handling by BOLT RUN: yaml2obj %p/Inputs/plt-got.yaml &> %t.exe -RUN: llvm-bolt %t.exe -o %t.bolt --print-disasm --print-only=_start/1 | \ -RUN: FileCheck %s +RUN: llvm-bolt %t.exe -o %t.bolt --print-disasm --print-only=_start/1 \ +RUN: 2>&1 | FileCheck %s CHECK: bl abort@PLT +CHECK: BOLT-WARNING: found unmarked AArch64 veneer diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def index 8428fa9..01d121b 100644 --- a/clang/include/clang/Basic/BuiltinsAMDGPU.def +++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def @@ -830,6 +830,15 @@ TARGET_BUILTIN(__builtin_amdgcn_perm_pk16_b4_u4, "V2UiUiUiV2Ui", "nc", "tensor-c TARGET_BUILTIN(__builtin_amdgcn_perm_pk16_b6_u4, "V3UiUiULiV2Ui", "nc", "tensor-cvt-lut-insts") TARGET_BUILTIN(__builtin_amdgcn_perm_pk16_b8_u4, "V4UiULiULiV2Ui", "nc", "tensor-cvt-lut-insts") +TARGET_BUILTIN(__builtin_amdgcn_add_max_i32, "iiiiIb", "nc", "add-min-max-insts") +TARGET_BUILTIN(__builtin_amdgcn_add_max_u32, "UiUiUiUiIb", "nc", "add-min-max-insts") +TARGET_BUILTIN(__builtin_amdgcn_add_min_i32, "iiiiIb", "nc", "add-min-max-insts") +TARGET_BUILTIN(__builtin_amdgcn_add_min_u32, "UiUiUiUiIb", "nc", "add-min-max-insts") +TARGET_BUILTIN(__builtin_amdgcn_pk_add_max_i16, "V2sV2sV2sV2sIb", "nc", "pk-add-min-max-insts") +TARGET_BUILTIN(__builtin_amdgcn_pk_add_max_u16, "V2UsV2UsV2UsV2UsIb", "nc", "pk-add-min-max-insts") +TARGET_BUILTIN(__builtin_amdgcn_pk_add_min_i16, "V2sV2sV2sV2sIb", "nc", "pk-add-min-max-insts") +TARGET_BUILTIN(__builtin_amdgcn_pk_add_min_u16, "V2UsV2UsV2UsV2UsIb", "nc", "pk-add-min-max-insts") + // GFX1250 WMMA builtins TARGET_BUILTIN(__builtin_amdgcn_wmma_f32_16x16x4_f32, "V8fIbV2fIbV2fIsV8fIbIb", "nc", "gfx1250-insts,wavefrontsize32") TARGET_BUILTIN(__builtin_amdgcn_wmma_f32_16x16x32_bf16, "V8fIbV16yIbV16yIsV8fIbIb", "nc", "gfx1250-insts,wavefrontsize32") diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 87b96c2..189798f 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -1321,15 +1321,11 @@ public: /// Callback to the parser to parse templated functions when needed. typedef void LateTemplateParserCB(void *P, LateParsedTemplate &LPT); - typedef void LateTemplateParserCleanupCB(void *P); LateTemplateParserCB *LateTemplateParser; - LateTemplateParserCleanupCB *LateTemplateParserCleanup; void *OpaqueParser; - void SetLateTemplateParser(LateTemplateParserCB *LTP, - LateTemplateParserCleanupCB *LTPCleanup, void *P) { + void SetLateTemplateParser(LateTemplateParserCB *LTP, void *P) { LateTemplateParser = LTP; - LateTemplateParserCleanup = LTPCleanup; OpaqueParser = P; } diff --git a/clang/lib/Analysis/FlowSensitive/Models/UncheckedStatusOrAccessModel.cpp b/clang/lib/Analysis/FlowSensitive/Models/UncheckedStatusOrAccessModel.cpp index f068be5..598d33a 100644 --- a/clang/lib/Analysis/FlowSensitive/Models/UncheckedStatusOrAccessModel.cpp +++ b/clang/lib/Analysis/FlowSensitive/Models/UncheckedStatusOrAccessModel.cpp @@ -137,6 +137,37 @@ static auto valueOperatorCall() { isStatusOrOperatorCallWithName("->"))); } +static clang::ast_matchers::TypeMatcher statusType() { + using namespace ::clang::ast_matchers; // NOLINT: Too many names + return hasCanonicalType(qualType(hasDeclaration(statusClass()))); +} + +static auto isComparisonOperatorCall(llvm::StringRef operator_name) { + using namespace ::clang::ast_matchers; // NOLINT: Too many names + return cxxOperatorCallExpr( + hasOverloadedOperatorName(operator_name), argumentCountIs(2), + hasArgument(0, anyOf(hasType(statusType()), hasType(statusOrType()))), + hasArgument(1, anyOf(hasType(statusType()), hasType(statusOrType())))); +} + +static auto isOkStatusCall() { + using namespace ::clang::ast_matchers; // NOLINT: Too many names + return callExpr(callee(functionDecl(hasName("::absl::OkStatus")))); +} + +static auto isNotOkStatusCall() { + using namespace ::clang::ast_matchers; // NOLINT: Too many names + return callExpr(callee(functionDecl(hasAnyName( + "::absl::AbortedError", "::absl::AlreadyExistsError", + "::absl::CancelledError", "::absl::DataLossError", + "::absl::DeadlineExceededError", "::absl::FailedPreconditionError", + "::absl::InternalError", "::absl::InvalidArgumentError", + "::absl::NotFoundError", "::absl::OutOfRangeError", + "::absl::PermissionDeniedError", "::absl::ResourceExhaustedError", + "::absl::UnauthenticatedError", "::absl::UnavailableError", + "::absl::UnimplementedError", "::absl::UnknownError")))); +} + static auto buildDiagnoseMatchSwitch(const UncheckedStatusOrAccessModelOptions &Options) { return CFGMatchSwitchBuilder<const Environment, @@ -312,6 +343,118 @@ static void transferStatusUpdateCall(const CXXMemberCallExpr *Expr, State.Env.setValue(locForOk(*ThisLoc), NewVal); } +static BoolValue *evaluateStatusEquality(RecordStorageLocation &LhsStatusLoc, + RecordStorageLocation &RhsStatusLoc, + Environment &Env) { + auto &A = Env.arena(); + // Logically, a Status object is composed of an error code that could take one + // of multiple possible values, including the "ok" value. We track whether a + // Status object has an "ok" value and represent this as an `ok` bit. Equality + // of Status objects compares their error codes. Therefore, merely comparing + // the `ok` bits isn't sufficient: when two Status objects are assigned non-ok + // error codes the equality of their respective error codes matters. Since we + // only track the `ok` bits, we can't make any conclusions about equality when + // we know that two Status objects have non-ok values. + + auto &LhsOkVal = valForOk(LhsStatusLoc, Env); + auto &RhsOkVal = valForOk(RhsStatusLoc, Env); + + auto &Res = Env.makeAtomicBoolValue(); + + // lhs && rhs => res (a.k.a. !res => !lhs || !rhs) + Env.assume(A.makeImplies(A.makeAnd(LhsOkVal.formula(), RhsOkVal.formula()), + Res.formula())); + // res => (lhs == rhs) + Env.assume(A.makeImplies( + Res.formula(), A.makeEquals(LhsOkVal.formula(), RhsOkVal.formula()))); + + return &Res; +} + +static BoolValue * +evaluateStatusOrEquality(RecordStorageLocation &LhsStatusOrLoc, + RecordStorageLocation &RhsStatusOrLoc, + Environment &Env) { + auto &A = Env.arena(); + // Logically, a StatusOr<T> object is composed of two values - a Status and a + // value of type T. Equality of StatusOr objects compares both values. + // Therefore, merely comparing the `ok` bits of the Status values isn't + // sufficient. When two StatusOr objects are engaged, the equality of their + // respective values of type T matters. Similarly, when two StatusOr objects + // have Status values that have non-ok error codes, the equality of the error + // codes matters. Since we only track the `ok` bits of the Status values, we + // can't make any conclusions about equality when we know that two StatusOr + // objects are engaged or when their Status values contain non-ok error codes. + auto &LhsOkVal = valForOk(locForStatus(LhsStatusOrLoc), Env); + auto &RhsOkVal = valForOk(locForStatus(RhsStatusOrLoc), Env); + auto &res = Env.makeAtomicBoolValue(); + + // res => (lhs == rhs) + Env.assume(A.makeImplies( + res.formula(), A.makeEquals(LhsOkVal.formula(), RhsOkVal.formula()))); + return &res; +} + +static BoolValue *evaluateEquality(const Expr *LhsExpr, const Expr *RhsExpr, + Environment &Env) { + // Check the type of both sides in case an operator== is added that admits + // different types. + if (isStatusOrType(LhsExpr->getType()) && + isStatusOrType(RhsExpr->getType())) { + auto *LhsStatusOrLoc = Env.get<RecordStorageLocation>(*LhsExpr); + if (LhsStatusOrLoc == nullptr) + return nullptr; + auto *RhsStatusOrLoc = Env.get<RecordStorageLocation>(*RhsExpr); + if (RhsStatusOrLoc == nullptr) + return nullptr; + + return evaluateStatusOrEquality(*LhsStatusOrLoc, *RhsStatusOrLoc, Env); + } + if (isStatusType(LhsExpr->getType()) && isStatusType(RhsExpr->getType())) { + auto *LhsStatusLoc = Env.get<RecordStorageLocation>(*LhsExpr); + if (LhsStatusLoc == nullptr) + return nullptr; + + auto *RhsStatusLoc = Env.get<RecordStorageLocation>(*RhsExpr); + if (RhsStatusLoc == nullptr) + return nullptr; + + return evaluateStatusEquality(*LhsStatusLoc, *RhsStatusLoc, Env); + } + return nullptr; +} + +static void transferComparisonOperator(const CXXOperatorCallExpr *Expr, + LatticeTransferState &State, + bool IsNegative) { + auto *LhsAndRhsVal = + evaluateEquality(Expr->getArg(0), Expr->getArg(1), State.Env); + if (LhsAndRhsVal == nullptr) + return; + + if (IsNegative) + State.Env.setValue(*Expr, State.Env.makeNot(*LhsAndRhsVal)); + else + State.Env.setValue(*Expr, *LhsAndRhsVal); +} + +static void transferOkStatusCall(const CallExpr *Expr, + const MatchFinder::MatchResult &, + LatticeTransferState &State) { + auto &OkVal = + initializeStatus(State.Env.getResultObjectLocation(*Expr), State.Env); + State.Env.assume(OkVal.formula()); +} + +static void transferNotOkStatusCall(const CallExpr *Expr, + const MatchFinder::MatchResult &, + LatticeTransferState &State) { + auto &OkVal = + initializeStatus(State.Env.getResultObjectLocation(*Expr), State.Env); + auto &A = State.Env.arena(); + State.Env.assume(A.makeNot(OkVal.formula())); +} + CFGMatchSwitch<LatticeTransferState> buildTransferMatchSwitch(ASTContext &Ctx, CFGMatchSwitchBuilder<LatticeTransferState> Builder) { @@ -325,6 +468,22 @@ buildTransferMatchSwitch(ASTContext &Ctx, transferStatusOkCall) .CaseOfCFGStmt<CXXMemberCallExpr>(isStatusMemberCallWithName("Update"), transferStatusUpdateCall) + .CaseOfCFGStmt<CXXOperatorCallExpr>( + isComparisonOperatorCall("=="), + [](const CXXOperatorCallExpr *Expr, const MatchFinder::MatchResult &, + LatticeTransferState &State) { + transferComparisonOperator(Expr, State, + /*IsNegative=*/false); + }) + .CaseOfCFGStmt<CXXOperatorCallExpr>( + isComparisonOperatorCall("!="), + [](const CXXOperatorCallExpr *Expr, const MatchFinder::MatchResult &, + LatticeTransferState &State) { + transferComparisonOperator(Expr, State, + /*IsNegative=*/true); + }) + .CaseOfCFGStmt<CallExpr>(isOkStatusCall(), transferOkStatusCall) + .CaseOfCFGStmt<CallExpr>(isNotOkStatusCall(), transferNotOkStatusCall) .Build(); } diff --git a/clang/lib/CIR/CodeGen/CIRGenAsm.cpp b/clang/lib/CIR/CodeGen/CIRGenAsm.cpp index 17dffb3..88a7e85 100644 --- a/clang/lib/CIR/CodeGen/CIRGenAsm.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenAsm.cpp @@ -117,9 +117,9 @@ mlir::LogicalResult CIRGenFunction::emitAsmStmt(const AsmStmt &s) { bool hasSideEffect = s.isVolatile() || s.getNumOutputs() == 0; - cir::InlineAsmOp ia = builder.create<cir::InlineAsmOp>( - getLoc(s.getAsmLoc()), resultType, operands, asmString, constraints, - hasSideEffect, inferFlavor(cgm, s), mlir::ArrayAttr()); + cir::InlineAsmOp ia = cir::InlineAsmOp::create( + builder, getLoc(s.getAsmLoc()), resultType, operands, asmString, + constraints, hasSideEffect, inferFlavor(cgm, s), mlir::ArrayAttr()); if (isGCCAsmGoto) { assert(!cir::MissingFeatures::asmGoto()); diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp b/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp index 670a431..75355ee 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp @@ -22,8 +22,8 @@ mlir::Value CIRGenBuilderTy::maybeBuildArrayDecay(mlir::Location loc, if (arrayTy) { const cir::PointerType flatPtrTy = getPointerTo(arrayTy.getElementType()); - return create<cir::CastOp>(loc, flatPtrTy, cir::CastKind::array_to_ptrdecay, - arrayPtr); + return cir::CastOp::create(*this, loc, flatPtrTy, + cir::CastKind::array_to_ptrdecay, arrayPtr); } assert(arrayPtrTy.getPointee() == eltTy && @@ -40,7 +40,7 @@ mlir::Value CIRGenBuilderTy::getArrayElement(mlir::Location arrayLocBegin, if (shouldDecay) basePtr = maybeBuildArrayDecay(arrayLocBegin, arrayPtr, eltTy); const mlir::Type flatPtrTy = basePtr.getType(); - return create<cir::PtrStrideOp>(arrayLocEnd, flatPtrTy, basePtr, idx); + return cir::PtrStrideOp::create(*this, arrayLocEnd, flatPtrTy, basePtr, idx); } cir::ConstantOp CIRGenBuilderTy::getConstInt(mlir::Location loc, @@ -60,14 +60,14 @@ cir::ConstantOp CIRGenBuilderTy::getConstInt(mlir::Location loc, cir::ConstantOp CIRGenBuilderTy::getConstInt(mlir::Location loc, mlir::Type t, uint64_t c) { assert(mlir::isa<cir::IntType>(t) && "expected cir::IntType"); - return create<cir::ConstantOp>(loc, cir::IntAttr::get(t, c)); + return cir::ConstantOp::create(*this, loc, cir::IntAttr::get(t, c)); } cir::ConstantOp clang::CIRGen::CIRGenBuilderTy::getConstFP(mlir::Location loc, mlir::Type t, llvm::APFloat fpVal) { assert(mlir::isa<cir::FPTypeInterface>(t) && "expected floating point type"); - return create<cir::ConstantOp>(loc, cir::FPAttr::get(t, fpVal)); + return cir::ConstantOp::create(*this, loc, cir::FPAttr::get(t, fpVal)); } void CIRGenBuilderTy::computeGlobalViewIndicesFromFlatOffset( diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp index 798e9d9..27c4d11 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp @@ -46,9 +46,9 @@ static RValue emitBuiltinBitOp(CIRGenFunction &cgf, const CallExpr *e, Op op; if constexpr (std::is_same_v<Op, cir::BitClzOp> || std::is_same_v<Op, cir::BitCtzOp>) - op = builder.create<Op>(cgf.getLoc(e->getSourceRange()), arg, poisonZero); + op = Op::create(builder, cgf.getLoc(e->getSourceRange()), arg, poisonZero); else - op = builder.create<Op>(cgf.getLoc(e->getSourceRange()), arg); + op = Op::create(builder, cgf.getLoc(e->getSourceRange()), arg); mlir::Value result = op.getResult(); mlir::Type exprTy = cgf.convertType(e->getType()); @@ -67,8 +67,8 @@ RValue CIRGenFunction::emitRotate(const CallExpr *e, bool isRotateLeft) { // to the type of input when necessary. assert(!cir::MissingFeatures::msvcBuiltins()); - auto r = builder.create<cir::RotateOp>(getLoc(e->getSourceRange()), input, - amount, isRotateLeft); + auto r = cir::RotateOp::create(builder, getLoc(e->getSourceRange()), input, + amount, isRotateLeft); return RValue::get(r); } @@ -227,14 +227,14 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID, return RValue::get(nullptr); mlir::Value argValue = emitCheckedArgForAssume(e->getArg(0)); - builder.create<cir::AssumeOp>(loc, argValue); + cir::AssumeOp::create(builder, loc, argValue); return RValue::get(nullptr); } case Builtin::BI__builtin_assume_separate_storage: { mlir::Value value0 = emitScalarExpr(e->getArg(0)); mlir::Value value1 = emitScalarExpr(e->getArg(1)); - builder.create<cir::AssumeSepStorageOp>(loc, value0, value1); + cir::AssumeSepStorageOp::create(builder, loc, value0, value1); return RValue::get(nullptr); } @@ -363,8 +363,8 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID, probability); } - auto result = builder.create<cir::ExpectOp>( - loc, argValue.getType(), argValue, expectedValue, probAttr); + auto result = cir::ExpectOp::create(builder, loc, argValue.getType(), + argValue, expectedValue, probAttr); return RValue::get(result); } @@ -375,7 +375,7 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID, case Builtin::BI_byteswap_ulong: case Builtin::BI_byteswap_uint64: { mlir::Value arg = emitScalarExpr(e->getArg(0)); - return RValue::get(builder.create<cir::ByteSwapOp>(loc, arg)); + return RValue::get(cir::ByteSwapOp::create(builder, loc, arg)); } case Builtin::BI__builtin_bitreverse8: @@ -383,7 +383,7 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID, case Builtin::BI__builtin_bitreverse32: case Builtin::BI__builtin_bitreverse64: { mlir::Value arg = emitScalarExpr(e->getArg(0)); - return RValue::get(builder.create<cir::BitReverseOp>(loc, arg)); + return RValue::get(cir::BitReverseOp::create(builder, loc, arg)); } case Builtin::BI__builtin_rotateleft8: diff --git a/clang/lib/CIR/CodeGen/CIRGenCXX.cpp b/clang/lib/CIR/CodeGen/CIRGenCXX.cpp index 171ce1c..ca421e5 100644 --- a/clang/lib/CIR/CodeGen/CIRGenCXX.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenCXX.cpp @@ -151,7 +151,7 @@ static void emitDeclDestroy(CIRGenFunction &cgf, const VarDecl *vd, // Don't confuse lexical cleanup. builder.clearInsertionPoint(); } else { - builder.create<cir::YieldOp>(addr.getLoc()); + cir::YieldOp::create(builder, addr.getLoc()); } } diff --git a/clang/lib/CIR/CodeGen/CIRGenCXXABI.cpp b/clang/lib/CIR/CodeGen/CIRGenCXXABI.cpp index eef3739..aa0182e 100644 --- a/clang/lib/CIR/CodeGen/CIRGenCXXABI.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenCXXABI.cpp @@ -70,8 +70,8 @@ cir::GlobalLinkageKind CIRGenCXXABI::getCXXDestructorLinkage( mlir::Value CIRGenCXXABI::loadIncomingCXXThis(CIRGenFunction &cgf) { ImplicitParamDecl *vd = getThisDecl(cgf); Address addr = cgf.getAddrOfLocalVar(vd); - return cgf.getBuilder().create<cir::LoadOp>( - cgf.getLoc(vd->getLocation()), addr.getElementType(), addr.getPointer()); + return cir::LoadOp::create(cgf.getBuilder(), cgf.getLoc(vd->getLocation()), + addr.getElementType(), addr.getPointer()); } void CIRGenCXXABI::setCXXABIThisValue(CIRGenFunction &cgf, diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.cpp b/clang/lib/CIR/CodeGen/CIRGenCall.cpp index 61072f0..88aef89 100644 --- a/clang/lib/CIR/CodeGen/CIRGenCall.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenCall.cpp @@ -690,6 +690,22 @@ void CallArg::copyInto(CIRGenFunction &cgf, Address addr, isUsed = true; } +mlir::Value CIRGenFunction::emitRuntimeCall(mlir::Location loc, + cir::FuncOp callee, + ArrayRef<mlir::Value> args) { + // TODO(cir): set the calling convention to this runtime call. + assert(!cir::MissingFeatures::opFuncCallingConv()); + + cir::CallOp call = builder.createCallOp(loc, callee, args); + assert(call->getNumResults() <= 1 && + "runtime functions have at most 1 result"); + + if (call->getNumResults() == 0) + return nullptr; + + return call->getResult(0); +} + void CIRGenFunction::emitCallArg(CallArgList &args, const clang::Expr *e, clang::QualType argType) { assert(argType->isReferenceType() == e->isGLValue() && diff --git a/clang/lib/CIR/CodeGen/CIRGenClass.cpp b/clang/lib/CIR/CodeGen/CIRGenClass.cpp index 89f4926..5046e09 100644 --- a/clang/lib/CIR/CodeGen/CIRGenClass.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenClass.cpp @@ -725,8 +725,9 @@ void CIRGenFunction::emitCXXAggrConstructorCall( // Emit the constructor call that will execute for every array element. mlir::Value arrayOp = builder.createPtrBitcast(arrayBase.getPointer(), arrayTy); - builder.create<cir::ArrayCtor>( - *currSrcLoc, arrayOp, [&](mlir::OpBuilder &b, mlir::Location loc) { + cir::ArrayCtor::create( + builder, *currSrcLoc, arrayOp, + [&](mlir::OpBuilder &b, mlir::Location loc) { mlir::BlockArgument arg = b.getInsertionBlock()->addArgument(ptrToElmType, loc); Address curAddr = Address(arg, elementType, eltAlignment); @@ -738,7 +739,7 @@ void CIRGenFunction::emitCXXAggrConstructorCall( emitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false, /*Delegating=*/false, currAVS, e); - builder.create<cir::YieldOp>(loc); + cir::YieldOp::create(builder, loc); }); } } diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp index 52021fc..9df88ad 100644 --- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp @@ -251,8 +251,8 @@ void CIRGenFunction::emitStoreThroughLValue(RValue src, LValue dst, const mlir::Location loc = dst.getVectorPointer().getLoc(); const mlir::Value vector = builder.createLoad(loc, dst.getVectorAddress()); - const mlir::Value newVector = builder.create<cir::VecInsertOp>( - loc, vector, src.getValue(), dst.getVectorIdx()); + const mlir::Value newVector = cir::VecInsertOp::create( + builder, loc, vector, src.getValue(), dst.getVectorIdx()); builder.createStore(loc, newVector, dst.getVectorAddress()); return; } @@ -615,8 +615,8 @@ RValue CIRGenFunction::emitLoadOfLValue(LValue lv, SourceLocation loc) { if (lv.isVectorElt()) { const mlir::Value load = builder.createLoad(getLoc(loc), lv.getVectorAddress()); - return RValue::get(builder.create<cir::VecExtractOp>(getLoc(loc), load, - lv.getVectorIdx())); + return RValue::get(cir::VecExtractOp::create(builder, getLoc(loc), load, + lv.getVectorIdx())); } cgm.errorNYI(loc, "emitLoadOfLValue"); @@ -671,8 +671,8 @@ static LValue emitFunctionDeclLValue(CIRGenFunction &cgf, const Expr *e, mlir::Type fnTy = funcOp.getFunctionType(); mlir::Type ptrTy = cir::PointerType::get(fnTy); - mlir::Value addr = cgf.getBuilder().create<cir::GetGlobalOp>( - loc, ptrTy, funcOp.getSymName()); + mlir::Value addr = cir::GetGlobalOp::create(cgf.getBuilder(), loc, ptrTy, + funcOp.getSymName()); if (funcOp.getFunctionType() != cgf.convertType(fd->getType())) { fnTy = cgf.convertType(fd->getType()); @@ -1685,8 +1685,8 @@ CIRGenCallee CIRGenFunction::emitDirectCallee(const GlobalDecl &gd) { mlir::OpBuilder::InsertionGuard guard(builder); builder.setInsertionPointToStart(cgm.getModule().getBody()); - clone = builder.create<cir::FuncOp>(calleeFunc.getLoc(), fdInlineName, - calleeFunc.getFunctionType()); + clone = cir::FuncOp::create(builder, calleeFunc.getLoc(), fdInlineName, + calleeFunc.getFunctionType()); clone.setLinkageAttr(cir::GlobalLinkageKindAttr::get( &cgm.getMLIRContext(), cir::GlobalLinkageKind::InternalLinkage)); clone.setSymVisibility("private"); @@ -1778,8 +1778,8 @@ RValue CIRGenFunction::emitCall(clang::QualType calleeTy, mlir::Operation *fn = callee.getFunctionPointer(); mlir::Value addr; if (auto funcOp = mlir::dyn_cast<cir::FuncOp>(fn)) { - addr = builder.create<cir::GetGlobalOp>( - getLoc(e->getSourceRange()), + addr = cir::GetGlobalOp::create( + builder, getLoc(e->getSourceRange()), cir::PointerType::get(funcOp.getFunctionType()), funcOp.getSymName()); } else { addr = fn->getResult(0); @@ -1820,10 +1820,12 @@ CIRGenCallee CIRGenFunction::emitCallee(const clang::Expr *e) { // Resolve direct calls. const auto *funcDecl = cast<FunctionDecl>(declRef->getDecl()); return emitDirectCallee(funcDecl); - } else if (isa<MemberExpr>(e)) { - cgm.errorNYI(e->getSourceRange(), - "emitCallee: call to member function is NYI"); - return {}; + } else if (auto me = dyn_cast<MemberExpr>(e)) { + if (const auto *fd = dyn_cast<FunctionDecl>(me->getMemberDecl())) { + emitIgnoredExpr(me->getBase()); + return emitDirectCallee(fd); + } + // Else fall through to the indirect reference handling below. } else if (auto *pde = dyn_cast<CXXPseudoDestructorExpr>(e)) { return CIRGenCallee::forPseudoDestructor(pde); } @@ -1996,9 +1998,9 @@ cir::IfOp CIRGenFunction::emitIfOnBoolExpr( // Emit the code with the fully general case. mlir::Value condV = emitOpOnBoolExpr(loc, cond); - return builder.create<cir::IfOp>(loc, condV, elseLoc.has_value(), - /*thenBuilder=*/thenBuilder, - /*elseBuilder=*/elseBuilder); + return cir::IfOp::create(builder, loc, condV, elseLoc.has_value(), + /*thenBuilder=*/thenBuilder, + /*elseBuilder=*/elseBuilder); } /// TODO(cir): see EmitBranchOnBoolExpr for extra ideas). @@ -2020,18 +2022,17 @@ mlir::Value CIRGenFunction::emitOpOnBoolExpr(mlir::Location loc, mlir::Value condV = emitOpOnBoolExpr(loc, condOp->getCond()); mlir::Value ternaryOpRes = - builder - .create<cir::TernaryOp>( - loc, condV, /*thenBuilder=*/ - [this, trueExpr](mlir::OpBuilder &b, mlir::Location loc) { - mlir::Value lhs = emitScalarExpr(trueExpr); - b.create<cir::YieldOp>(loc, lhs); - }, - /*elseBuilder=*/ - [this, falseExpr](mlir::OpBuilder &b, mlir::Location loc) { - mlir::Value rhs = emitScalarExpr(falseExpr); - b.create<cir::YieldOp>(loc, rhs); - }) + cir::TernaryOp::create( + builder, loc, condV, /*thenBuilder=*/ + [this, trueExpr](mlir::OpBuilder &b, mlir::Location loc) { + mlir::Value lhs = emitScalarExpr(trueExpr); + cir::YieldOp::create(b, loc, lhs); + }, + /*elseBuilder=*/ + [this, falseExpr](mlir::OpBuilder &b, mlir::Location loc) { + mlir::Value rhs = emitScalarExpr(falseExpr); + cir::YieldOp::create(b, loc, rhs); + }) .getResult(); return emitScalarConversion(ternaryOpRes, condOp->getType(), @@ -2211,8 +2212,8 @@ Address CIRGenFunction::emitLoadOfReference(LValue refLVal, mlir::Location loc, cgm.errorNYI(loc, "load of volatile reference"); cir::LoadOp load = - builder.create<cir::LoadOp>(loc, refLVal.getAddress().getElementType(), - refLVal.getAddress().getPointer()); + cir::LoadOp::create(builder, loc, refLVal.getAddress().getElementType(), + refLVal.getAddress().getPointer()); assert(!cir::MissingFeatures::opTBAA()); diff --git a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp index d8f4943..047f359 100644 --- a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp @@ -390,7 +390,7 @@ ComplexExprEmitter::VisitImaginaryLiteral(const ImaginaryLiteral *il) { } auto complexAttr = cir::ConstComplexAttr::get(realValueAttr, imagValueAttr); - return builder.create<cir::ConstantOp>(loc, complexAttr); + return cir::ConstantOp::create(builder, loc, complexAttr); } mlir::Value ComplexExprEmitter::VisitCallExpr(const CallExpr *e) { @@ -601,7 +601,7 @@ mlir::Value ComplexExprEmitter::emitBinAdd(const BinOpInfo &op) { if (mlir::isa<cir::ComplexType>(op.lhs.getType()) && mlir::isa<cir::ComplexType>(op.rhs.getType())) - return builder.create<cir::ComplexAddOp>(op.loc, op.lhs, op.rhs); + return cir::ComplexAddOp::create(builder, op.loc, op.lhs, op.rhs); if (mlir::isa<cir::ComplexType>(op.lhs.getType())) { mlir::Value real = builder.createComplexReal(op.loc, op.lhs); @@ -623,7 +623,7 @@ mlir::Value ComplexExprEmitter::emitBinSub(const BinOpInfo &op) { if (mlir::isa<cir::ComplexType>(op.lhs.getType()) && mlir::isa<cir::ComplexType>(op.rhs.getType())) - return builder.create<cir::ComplexSubOp>(op.loc, op.lhs, op.rhs); + return cir::ComplexSubOp::create(builder, op.loc, op.lhs, op.rhs); if (mlir::isa<cir::ComplexType>(op.lhs.getType())) { mlir::Value real = builder.createComplexReal(op.loc, op.lhs); @@ -664,7 +664,8 @@ mlir::Value ComplexExprEmitter::emitBinMul(const BinOpInfo &op) { mlir::isa<cir::ComplexType>(op.rhs.getType())) { cir::ComplexRangeKind rangeKind = getComplexRangeAttr(op.fpFeatures.getComplexRange()); - return builder.create<cir::ComplexMulOp>(op.loc, op.lhs, op.rhs, rangeKind); + return cir::ComplexMulOp::create(builder, op.loc, op.lhs, op.rhs, + rangeKind); } if (mlir::isa<cir::ComplexType>(op.lhs.getType())) { @@ -968,23 +969,22 @@ mlir::Value ComplexExprEmitter::VisitAbstractConditionalOperator( Expr *cond = e->getCond()->IgnoreParens(); mlir::Value condValue = cgf.evaluateExprAsBool(cond); - return builder - .create<cir::TernaryOp>( - loc, condValue, - /*thenBuilder=*/ - [&](mlir::OpBuilder &b, mlir::Location loc) { - eval.beginEvaluation(); - mlir::Value trueValue = Visit(e->getTrueExpr()); - b.create<cir::YieldOp>(loc, trueValue); - eval.endEvaluation(); - }, - /*elseBuilder=*/ - [&](mlir::OpBuilder &b, mlir::Location loc) { - eval.beginEvaluation(); - mlir::Value falseValue = Visit(e->getFalseExpr()); - b.create<cir::YieldOp>(loc, falseValue); - eval.endEvaluation(); - }) + return cir::TernaryOp::create( + builder, loc, condValue, + /*thenBuilder=*/ + [&](mlir::OpBuilder &b, mlir::Location loc) { + eval.beginEvaluation(); + mlir::Value trueValue = Visit(e->getTrueExpr()); + cir::YieldOp::create(b, loc, trueValue); + eval.endEvaluation(); + }, + /*elseBuilder=*/ + [&](mlir::OpBuilder &b, mlir::Location loc) { + eval.beginEvaluation(); + mlir::Value falseValue = Visit(e->getFalseExpr()); + cir::YieldOp::create(b, loc, falseValue); + eval.endEvaluation(); + }) .getResult(); } diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp index 800262a..8f05014 100644 --- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp @@ -179,8 +179,23 @@ bool ConstantAggregateBuilder::add(mlir::TypedAttr typedAttr, CharUnits offset, } // Uncommon case: constant overlaps what we've already created. - cgm.errorNYI("overlapping constants"); - return false; + std::optional<size_t> firstElemToReplace = splitAt(offset); + if (!firstElemToReplace) + return false; + + CharUnits cSize = getSize(typedAttr); + std::optional<size_t> lastElemToReplace = splitAt(offset + cSize); + if (!lastElemToReplace) + return false; + + assert((firstElemToReplace == lastElemToReplace || allowOverwrite) && + "unexpectedly overwriting field"); + + Element newElt(typedAttr, offset); + replace(elements, *firstElemToReplace, *lastElemToReplace, {newElt}); + size = std::max(size, offset + cSize); + naturalLayout = false; + return true; } bool ConstantAggregateBuilder::addBits(llvm::APInt bits, uint64_t offsetInBits, diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp index 33eb748..db6878d 100644 --- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp @@ -164,22 +164,22 @@ public: mlir::Value VisitIntegerLiteral(const IntegerLiteral *e) { mlir::Type type = cgf.convertType(e->getType()); - return builder.create<cir::ConstantOp>( - cgf.getLoc(e->getExprLoc()), cir::IntAttr::get(type, e->getValue())); + return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()), + cir::IntAttr::get(type, e->getValue())); } mlir::Value VisitFloatingLiteral(const FloatingLiteral *e) { mlir::Type type = cgf.convertType(e->getType()); assert(mlir::isa<cir::FPTypeInterface>(type) && "expect floating-point type"); - return builder.create<cir::ConstantOp>( - cgf.getLoc(e->getExprLoc()), cir::FPAttr::get(type, e->getValue())); + return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()), + cir::FPAttr::get(type, e->getValue())); } mlir::Value VisitCharacterLiteral(const CharacterLiteral *e) { mlir::Type ty = cgf.convertType(e->getType()); auto init = cir::IntAttr::get(ty, e->getValue()); - return builder.create<cir::ConstantOp>(cgf.getLoc(e->getExprLoc()), init); + return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()), init); } mlir::Value VisitCXXBoolLiteralExpr(const CXXBoolLiteralExpr *e) { @@ -227,7 +227,7 @@ public: const mlir::Location loc = cgf.getLoc(e->getSourceRange()); const mlir::Value vecValue = Visit(e->getBase()); const mlir::Value indexValue = Visit(e->getIdx()); - return cgf.builder.create<cir::VecExtractOp>(loc, vecValue, indexValue); + return cir::VecExtractOp::create(cgf.builder, loc, vecValue, indexValue); } // Just load the lvalue formed by the subscript expression. return emitLoadOfLValue(e); @@ -238,8 +238,8 @@ public: // The undocumented form of __builtin_shufflevector. mlir::Value inputVec = Visit(e->getExpr(0)); mlir::Value indexVec = Visit(e->getExpr(1)); - return cgf.builder.create<cir::VecShuffleDynamicOp>( - cgf.getLoc(e->getSourceRange()), inputVec, indexVec); + return cir::VecShuffleDynamicOp::create( + cgf.builder, cgf.getLoc(e->getSourceRange()), inputVec, indexVec); } mlir::Value vec1 = Visit(e->getExpr(0)); @@ -257,9 +257,10 @@ public: .getSExtValue())); } - return cgf.builder.create<cir::VecShuffleOp>( - cgf.getLoc(e->getSourceRange()), cgf.convertType(e->getType()), vec1, - vec2, cgf.builder.getArrayAttr(indices)); + return cir::VecShuffleOp::create(cgf.builder, + cgf.getLoc(e->getSourceRange()), + cgf.convertType(e->getType()), vec1, vec2, + cgf.builder.getArrayAttr(indices)); } mlir::Value VisitConvertVectorExpr(ConvertVectorExpr *e) { @@ -296,8 +297,8 @@ public: mlir::Value emitFloatToBoolConversion(mlir::Value src, mlir::Location loc) { cir::BoolType boolTy = builder.getBoolTy(); - return builder.create<cir::CastOp>(loc, boolTy, - cir::CastKind::float_to_bool, src); + return cir::CastOp::create(builder, loc, boolTy, + cir::CastKind::float_to_bool, src); } mlir::Value emitIntToBoolConversion(mlir::Value srcVal, mlir::Location loc) { @@ -307,8 +308,8 @@ public: // TODO: optimize this common case here or leave it for later // CIR passes? cir::BoolType boolTy = builder.getBoolTy(); - return builder.create<cir::CastOp>(loc, boolTy, cir::CastKind::int_to_bool, - srcVal); + return cir::CastOp::create(builder, loc, boolTy, cir::CastKind::int_to_bool, + srcVal); } /// Convert the specified expression value to a boolean (!cir.bool) truth @@ -411,7 +412,8 @@ public: } assert(castKind.has_value() && "Internal error: CastKind not set."); - return builder.create<cir::CastOp>(src.getLoc(), fullDstTy, *castKind, src); + return cir::CastOp::create(builder, src.getLoc(), fullDstTy, *castKind, + src); } mlir::Value @@ -658,9 +660,9 @@ public: mlir::Value emitUnaryOp(const UnaryOperator *e, cir::UnaryOpKind kind, mlir::Value input, bool nsw = false) { - return builder.create<cir::UnaryOp>( - cgf.getLoc(e->getSourceRange().getBegin()), input.getType(), kind, - input, nsw); + return cir::UnaryOp::create(builder, + cgf.getLoc(e->getSourceRange().getBegin()), + input.getType(), kind, input, nsw); } mlir::Value VisitUnaryNot(const UnaryOperator *e) { @@ -967,9 +969,9 @@ public: } else { // Other kinds of vectors. Element-wise comparison returning // a vector. - result = builder.create<cir::VecCmpOp>( - cgf.getLoc(boInfo.loc), cgf.convertType(boInfo.fullType), kind, - boInfo.lhs, boInfo.rhs); + result = cir::VecCmpOp::create(builder, cgf.getLoc(boInfo.loc), + cgf.convertType(boInfo.fullType), kind, + boInfo.lhs, boInfo.rhs); } } else if (boInfo.isFixedPointOp()) { assert(!cir::MissingFeatures::fixedPointType()); @@ -991,7 +993,7 @@ public: assert(e->getOpcode() == BO_EQ || e->getOpcode() == BO_NE); BinOpInfo boInfo = emitBinOps(e); - result = builder.create<cir::CmpOp>(loc, kind, boInfo.lhs, boInfo.rhs); + result = cir::CmpOp::create(builder, loc, kind, boInfo.lhs, boInfo.rhs); } return emitScalarConversion(result, cgf.getContext().BoolTy, e->getType(), @@ -1093,8 +1095,8 @@ public: CIRGenFunction::ConditionalEvaluation eval(cgf); mlir::Value lhsCondV = cgf.evaluateExprAsBool(e->getLHS()); - auto resOp = builder.create<cir::TernaryOp>( - loc, lhsCondV, /*trueBuilder=*/ + auto resOp = cir::TernaryOp::create( + builder, loc, lhsCondV, /*trueBuilder=*/ [&](mlir::OpBuilder &b, mlir::Location loc) { CIRGenFunction::LexicalScope lexScope{cgf, loc, b.getInsertionBlock()}; @@ -1139,8 +1141,8 @@ public: CIRGenFunction::ConditionalEvaluation eval(cgf); mlir::Value lhsCondV = cgf.evaluateExprAsBool(e->getLHS()); - auto resOp = builder.create<cir::TernaryOp>( - loc, lhsCondV, /*trueBuilder=*/ + auto resOp = cir::TernaryOp::create( + builder, loc, lhsCondV, /*trueBuilder=*/ [&](mlir::OpBuilder &b, mlir::Location loc) { CIRGenFunction::LexicalScope lexScope{cgf, loc, b.getInsertionBlock()}; @@ -1566,8 +1568,9 @@ static mlir::Value emitPointerArithmetic(CIRGenFunction &cgf, } assert(!cir::MissingFeatures::sanitizers()); - return cgf.getBuilder().create<cir::PtrStrideOp>( - cgf.getLoc(op.e->getExprLoc()), pointer.getType(), pointer, index); + return cir::PtrStrideOp::create(cgf.getBuilder(), + cgf.getLoc(op.e->getExprLoc()), + pointer.getType(), pointer, index); } mlir::Value ScalarExprEmitter::emitMul(const BinOpInfo &ops) { @@ -1609,19 +1612,19 @@ mlir::Value ScalarExprEmitter::emitMul(const BinOpInfo &ops) { return nullptr; } - return builder.create<cir::BinOp>(cgf.getLoc(ops.loc), - cgf.convertType(ops.fullType), - cir::BinOpKind::Mul, ops.lhs, ops.rhs); + return cir::BinOp::create(builder, cgf.getLoc(ops.loc), + cgf.convertType(ops.fullType), cir::BinOpKind::Mul, + ops.lhs, ops.rhs); } mlir::Value ScalarExprEmitter::emitDiv(const BinOpInfo &ops) { - return builder.create<cir::BinOp>(cgf.getLoc(ops.loc), - cgf.convertType(ops.fullType), - cir::BinOpKind::Div, ops.lhs, ops.rhs); + return cir::BinOp::create(builder, cgf.getLoc(ops.loc), + cgf.convertType(ops.fullType), cir::BinOpKind::Div, + ops.lhs, ops.rhs); } mlir::Value ScalarExprEmitter::emitRem(const BinOpInfo &ops) { - return builder.create<cir::BinOp>(cgf.getLoc(ops.loc), - cgf.convertType(ops.fullType), - cir::BinOpKind::Rem, ops.lhs, ops.rhs); + return cir::BinOp::create(builder, cgf.getLoc(ops.loc), + cgf.convertType(ops.fullType), cir::BinOpKind::Rem, + ops.lhs, ops.rhs); } mlir::Value ScalarExprEmitter::emitAdd(const BinOpInfo &ops) { @@ -1668,8 +1671,8 @@ mlir::Value ScalarExprEmitter::emitAdd(const BinOpInfo &ops) { return {}; } - return builder.create<cir::BinOp>(loc, cgf.convertType(ops.fullType), - cir::BinOpKind::Add, ops.lhs, ops.rhs); + return cir::BinOp::create(builder, loc, cgf.convertType(ops.fullType), + cir::BinOpKind::Add, ops.lhs, ops.rhs); } mlir::Value ScalarExprEmitter::emitSub(const BinOpInfo &ops) { @@ -1716,9 +1719,9 @@ mlir::Value ScalarExprEmitter::emitSub(const BinOpInfo &ops) { return {}; } - return builder.create<cir::BinOp>(cgf.getLoc(ops.loc), - cgf.convertType(ops.fullType), - cir::BinOpKind::Sub, ops.lhs, ops.rhs); + return cir::BinOp::create(builder, cgf.getLoc(ops.loc), + cgf.convertType(ops.fullType), + cir::BinOpKind::Sub, ops.lhs, ops.rhs); } // If the RHS is not a pointer, then we have normal pointer @@ -1796,19 +1799,19 @@ mlir::Value ScalarExprEmitter::emitShr(const BinOpInfo &ops) { } mlir::Value ScalarExprEmitter::emitAnd(const BinOpInfo &ops) { - return builder.create<cir::BinOp>(cgf.getLoc(ops.loc), - cgf.convertType(ops.fullType), - cir::BinOpKind::And, ops.lhs, ops.rhs); + return cir::BinOp::create(builder, cgf.getLoc(ops.loc), + cgf.convertType(ops.fullType), cir::BinOpKind::And, + ops.lhs, ops.rhs); } mlir::Value ScalarExprEmitter::emitXor(const BinOpInfo &ops) { - return builder.create<cir::BinOp>(cgf.getLoc(ops.loc), - cgf.convertType(ops.fullType), - cir::BinOpKind::Xor, ops.lhs, ops.rhs); + return cir::BinOp::create(builder, cgf.getLoc(ops.loc), + cgf.convertType(ops.fullType), cir::BinOpKind::Xor, + ops.lhs, ops.rhs); } mlir::Value ScalarExprEmitter::emitOr(const BinOpInfo &ops) { - return builder.create<cir::BinOp>(cgf.getLoc(ops.loc), - cgf.convertType(ops.fullType), - cir::BinOpKind::Or, ops.lhs, ops.rhs); + return cir::BinOp::create(builder, cgf.getLoc(ops.loc), + cgf.convertType(ops.fullType), cir::BinOpKind::Or, + ops.lhs, ops.rhs); } // Emit code for an explicit or implicit cast. Implicit @@ -2011,9 +2014,9 @@ mlir::Value ScalarExprEmitter::VisitCastExpr(CastExpr *ce) { case CK_VectorSplat: { // Create a vector object and fill all elements with the same scalar value. assert(destTy->isVectorType() && "CK_VectorSplat to non-vector type"); - return builder.create<cir::VecSplatOp>( - cgf.getLoc(subExpr->getSourceRange()), cgf.convertType(destTy), - Visit(subExpr)); + return cir::VecSplatOp::create(builder, + cgf.getLoc(subExpr->getSourceRange()), + cgf.convertType(destTy), Visit(subExpr)); } case CK_FunctionToPointerDecay: return cgf.emitLValue(subExpr).getPointer(); @@ -2073,8 +2076,9 @@ mlir::Value ScalarExprEmitter::VisitInitListExpr(InitListExpr *e) { vectorType.getSize() - numInitElements, zeroValue); } - return cgf.getBuilder().create<cir::VecCreateOp>( - cgf.getLoc(e->getSourceRange()), vectorType, elements); + return cir::VecCreateOp::create(cgf.getBuilder(), + cgf.getLoc(e->getSourceRange()), vectorType, + elements); } // C++11 value-initialization for the scalar. @@ -2310,8 +2314,8 @@ mlir::Value ScalarExprEmitter::VisitAbstractConditionalOperator( mlir::Value condValue = Visit(condExpr); mlir::Value lhsValue = Visit(lhsExpr); mlir::Value rhsValue = Visit(rhsExpr); - return builder.create<cir::VecTernaryOp>(loc, condValue, lhsValue, - rhsValue); + return cir::VecTernaryOp::create(builder, loc, condValue, lhsValue, + rhsValue); } // If this is a really simple expression (like x ? 4 : 5), emit this as a @@ -2354,7 +2358,7 @@ mlir::Value ScalarExprEmitter::VisitAbstractConditionalOperator( if (branch) { yieldTy = branch.getType(); - b.create<cir::YieldOp>(loc, branch); + cir::YieldOp::create(b, loc, branch); } else { // If LHS or RHS is a throw or void expression we need to patch // arms as to properly match yield types. @@ -2362,17 +2366,16 @@ mlir::Value ScalarExprEmitter::VisitAbstractConditionalOperator( } }; - mlir::Value result = builder - .create<cir::TernaryOp>( - loc, condV, - /*trueBuilder=*/ - [&](mlir::OpBuilder &b, mlir::Location loc) { - emitBranch(b, loc, lhsExpr); - }, - /*falseBuilder=*/ - [&](mlir::OpBuilder &b, mlir::Location loc) { - emitBranch(b, loc, rhsExpr); - }) + mlir::Value result = cir::TernaryOp::create( + builder, loc, condV, + /*trueBuilder=*/ + [&](mlir::OpBuilder &b, mlir::Location loc) { + emitBranch(b, loc, lhsExpr); + }, + /*falseBuilder=*/ + [&](mlir::OpBuilder &b, mlir::Location loc) { + emitBranch(b, loc, rhsExpr); + }) .getResult(); if (!insertPoints.empty()) { @@ -2387,10 +2390,10 @@ mlir::Value ScalarExprEmitter::VisitAbstractConditionalOperator( // Block does not return: build empty yield. if (mlir::isa<cir::VoidType>(yieldTy)) { - builder.create<cir::YieldOp>(loc); + cir::YieldOp::create(builder, loc); } else { // Block returns: set null yield value. mlir::Value op0 = builder.getNullValue(yieldTy, loc); - builder.create<cir::YieldOp>(loc, op0); + cir::YieldOp::create(builder, loc, op0); } } } diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp index d3c0d9f..58feb36 100644 --- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp @@ -264,11 +264,11 @@ void CIRGenFunction::LexicalScope::cleanup() { // If we now have one after `applyCleanup`, hook it up properly. if (!cleanupBlock && localScope->getCleanupBlock(builder)) { cleanupBlock = localScope->getCleanupBlock(builder); - builder.create<cir::BrOp>(insPt->back().getLoc(), cleanupBlock); + cir::BrOp::create(builder, insPt->back().getLoc(), cleanupBlock); if (!cleanupBlock->mightHaveTerminator()) { mlir::OpBuilder::InsertionGuard guard(builder); builder.setInsertionPointToEnd(cleanupBlock); - builder.create<cir::YieldOp>(localScope->endLoc); + cir::YieldOp::create(builder, localScope->endLoc); } } @@ -286,7 +286,7 @@ void CIRGenFunction::LexicalScope::cleanup() { } } - builder.create<cir::BrOp>(*returnLoc, returnBlock); + cir::BrOp::create(builder, *returnLoc, returnBlock); return; } } @@ -298,8 +298,8 @@ void CIRGenFunction::LexicalScope::cleanup() { // Ternary ops have to deal with matching arms for yielding types // and do return a value, it must do its own cir.yield insertion. if (!localScope->isTernary() && !insPt->mightHaveTerminator()) { - !retVal ? builder.create<cir::YieldOp>(localScope->endLoc) - : builder.create<cir::YieldOp>(localScope->endLoc, retVal); + !retVal ? cir::YieldOp::create(builder, localScope->endLoc) + : cir::YieldOp::create(builder, localScope->endLoc, retVal); } }; @@ -331,7 +331,7 @@ void CIRGenFunction::LexicalScope::cleanup() { // If there's a cleanup block, branch to it, nothing else to do. if (cleanupBlock) { - builder.create<cir::BrOp>(curBlock->back().getLoc(), cleanupBlock); + cir::BrOp::create(builder, curBlock->back().getLoc(), cleanupBlock); return; } @@ -349,12 +349,12 @@ cir::ReturnOp CIRGenFunction::LexicalScope::emitReturn(mlir::Location loc) { assert(fn && "emitReturn from non-function"); if (!fn.getFunctionType().hasVoidReturn()) { // Load the value from `__retval` and return it via the `cir.return` op. - auto value = builder.create<cir::LoadOp>( - loc, fn.getFunctionType().getReturnType(), *cgf.fnRetAlloca); - return builder.create<cir::ReturnOp>(loc, - llvm::ArrayRef(value.getResult())); + auto value = cir::LoadOp::create( + builder, loc, fn.getFunctionType().getReturnType(), *cgf.fnRetAlloca); + return cir::ReturnOp::create(builder, loc, + llvm::ArrayRef(value.getResult())); } - return builder.create<cir::ReturnOp>(loc); + return cir::ReturnOp::create(builder, loc); } // This is copied from CodeGenModule::MayDropFunctionReturn. This is a @@ -389,9 +389,9 @@ void CIRGenFunction::LexicalScope::emitImplicitReturn() { if (shouldEmitUnreachable) { assert(!cir::MissingFeatures::sanitizers()); if (cgf.cgm.getCodeGenOpts().OptimizationLevel == 0) - builder.create<cir::TrapOp>(localScope->endLoc); + cir::TrapOp::create(builder, localScope->endLoc); else - builder.create<cir::UnreachableOp>(localScope->endLoc); + cir::UnreachableOp::create(builder, localScope->endLoc); builder.clearInsertionPoint(); return; } @@ -561,8 +561,8 @@ cir::FuncOp CIRGenFunction::generateCode(clang::GlobalDecl gd, cir::FuncOp fn, if (!clone) { mlir::OpBuilder::InsertionGuard guard(builder); builder.setInsertionPoint(fn); - clone = builder.create<cir::FuncOp>(fn.getLoc(), fdInlineName, - fn.getFunctionType()); + clone = cir::FuncOp::create(builder, fn.getLoc(), fdInlineName, + fn.getFunctionType()); clone.setLinkage(cir::GlobalLinkageKind::InternalLinkage); clone.setSymVisibility("private"); clone.setInlineKind(cir::InlineKind::AlwaysInline); diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h index e3b9b6a..5f9dbdc 100644 --- a/clang/lib/CIR/CodeGen/CIRGenFunction.h +++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h @@ -1461,6 +1461,9 @@ public: void emitReturnOfRValue(mlir::Location loc, RValue rv, QualType ty); + mlir::Value emitRuntimeCall(mlir::Location loc, cir::FuncOp callee, + llvm::ArrayRef<mlir::Value> args = {}); + /// Emit the computation of the specified expression of scalar type. mlir::Value emitScalarExpr(const clang::Expr *e); diff --git a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp index e620310..f7c4d18 100644 --- a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp @@ -1809,8 +1809,8 @@ CIRGenItaniumCXXABI::getVTableAddressPoint(BaseSubobject base, mlir::OpBuilder &builder = cgm.getBuilder(); auto vtablePtrTy = cir::VPtrType::get(builder.getContext()); - return builder.create<cir::VTableAddrPointOp>( - cgm.getLoc(vtableClass->getSourceRange()), vtablePtrTy, + return cir::VTableAddrPointOp::create( + builder, cgm.getLoc(vtableClass->getSourceRange()), vtablePtrTy, mlir::FlatSymbolRefAttr::get(vtable.getSymNameAttr()), cir::AddressPointAttr::get(cgm.getBuilder().getContext(), addressPoint.VTableIndex, @@ -1874,6 +1874,15 @@ static cir::FuncOp getBadCastFn(CIRGenFunction &cgf) { return cgf.cgm.createRuntimeFunction(fnTy, "__cxa_bad_cast"); } +static void emitCallToBadCast(CIRGenFunction &cgf, mlir::Location loc) { + // TODO(cir): set the calling convention to the runtime function. + assert(!cir::MissingFeatures::opFuncCallingConv()); + + cgf.emitRuntimeCall(loc, getBadCastFn(cgf)); + cir::UnreachableOp::create(cgf.getBuilder(), loc); + cgf.getBuilder().clearInsertionPoint(); +} + // TODO(cir): This could be shared with classic codegen. static CharUnits computeOffsetHint(ASTContext &astContext, const CXXRecordDecl *src, @@ -1959,6 +1968,136 @@ static Address emitDynamicCastToVoid(CIRGenFunction &cgf, mlir::Location loc, return Address{ptr, src.getAlignment()}; } +static mlir::Value emitExactDynamicCast(CIRGenItaniumCXXABI &abi, + CIRGenFunction &cgf, mlir::Location loc, + QualType srcRecordTy, + QualType destRecordTy, + cir::PointerType destCIRTy, + bool isRefCast, Address src) { + // Find all the inheritance paths from SrcRecordTy to DestRecordTy. + const CXXRecordDecl *srcDecl = srcRecordTy->getAsCXXRecordDecl(); + const CXXRecordDecl *destDecl = destRecordTy->getAsCXXRecordDecl(); + CXXBasePaths paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true, + /*DetectVirtual=*/false); + (void)destDecl->isDerivedFrom(srcDecl, paths); + + // Find an offset within `destDecl` where a `srcDecl` instance and its vptr + // might appear. + std::optional<CharUnits> offset; + for (const CXXBasePath &path : paths) { + // dynamic_cast only finds public inheritance paths. + if (path.Access != AS_public) + continue; + + CharUnits pathOffset; + for (const CXXBasePathElement &pathElement : path) { + // Find the offset along this inheritance step. + const CXXRecordDecl *base = + pathElement.Base->getType()->getAsCXXRecordDecl(); + if (pathElement.Base->isVirtual()) { + // For a virtual base class, we know that the derived class is exactly + // destDecl, so we can use the vbase offset from its layout. + const ASTRecordLayout &layout = + cgf.getContext().getASTRecordLayout(destDecl); + pathOffset = layout.getVBaseClassOffset(base); + } else { + const ASTRecordLayout &layout = + cgf.getContext().getASTRecordLayout(pathElement.Class); + pathOffset += layout.getBaseClassOffset(base); + } + } + + if (!offset) { + offset = pathOffset; + } else if (offset != pathOffset) { + // base appears in at least two different places. Find the most-derived + // object and see if it's a DestDecl. Note that the most-derived object + // must be at least as aligned as this base class subobject, and must + // have a vptr at offset 0. + src = emitDynamicCastToVoid(cgf, loc, srcRecordTy, src); + srcDecl = destDecl; + offset = CharUnits::Zero(); + break; + } + } + + CIRGenBuilderTy &builder = cgf.getBuilder(); + + if (!offset) { + // If there are no public inheritance paths, the cast always fails. + mlir::Value nullPtrValue = builder.getNullPtr(destCIRTy, loc); + if (isRefCast) { + mlir::Region *currentRegion = builder.getBlock()->getParent(); + emitCallToBadCast(cgf, loc); + + // The call to bad_cast will terminate the block. Create a new block to + // hold any follow up code. + builder.createBlock(currentRegion, currentRegion->end()); + } + + return nullPtrValue; + } + + // Compare the vptr against the expected vptr for the destination type at + // this offset. Note that we do not know what type src points to in the case + // where the derived class multiply inherits from the base class so we can't + // use getVTablePtr, so we load the vptr directly instead. + + mlir::Value expectedVPtr = + abi.getVTableAddressPoint(BaseSubobject(srcDecl, *offset), destDecl); + + // TODO(cir): handle address space here. + assert(!cir::MissingFeatures::addressSpace()); + mlir::Type vptrTy = expectedVPtr.getType(); + mlir::Type vptrPtrTy = builder.getPointerTo(vptrTy); + Address srcVPtrPtr(builder.createBitcast(src.getPointer(), vptrPtrTy), + src.getAlignment()); + mlir::Value srcVPtr = builder.createLoad(loc, srcVPtrPtr); + + // TODO(cir): decorate SrcVPtr with TBAA info. + assert(!cir::MissingFeatures::opTBAA()); + + mlir::Value success = + builder.createCompare(loc, cir::CmpOpKind::eq, srcVPtr, expectedVPtr); + + auto emitCastResult = [&] { + if (offset->isZero()) + return builder.createBitcast(src.getPointer(), destCIRTy); + + // TODO(cir): handle address space here. + assert(!cir::MissingFeatures::addressSpace()); + mlir::Type u8PtrTy = builder.getUInt8PtrTy(); + + mlir::Value strideToApply = + builder.getConstInt(loc, builder.getUInt64Ty(), -offset->getQuantity()); + mlir::Value srcU8Ptr = builder.createBitcast(src.getPointer(), u8PtrTy); + mlir::Value resultU8Ptr = cir::PtrStrideOp::create(builder, loc, u8PtrTy, + srcU8Ptr, strideToApply); + return builder.createBitcast(resultU8Ptr, destCIRTy); + }; + + if (isRefCast) { + mlir::Value failed = builder.createNot(success); + cir::IfOp::create(builder, loc, failed, /*withElseRegion=*/false, + [&](mlir::OpBuilder &, mlir::Location) { + emitCallToBadCast(cgf, loc); + }); + return emitCastResult(); + } + + return cir::TernaryOp::create( + builder, loc, success, + [&](mlir::OpBuilder &, mlir::Location) { + auto result = emitCastResult(); + builder.createYield(loc, result); + }, + [&](mlir::OpBuilder &, mlir::Location) { + mlir::Value nullPtrValue = builder.getNullPtr(destCIRTy, loc); + builder.createYield(loc, nullPtrValue); + }) + .getResult(); +} + static cir::DynamicCastInfoAttr emitDynamicCastInfo(CIRGenFunction &cgf, mlir::Location loc, QualType srcRecordTy, @@ -2000,8 +2139,27 @@ mlir::Value CIRGenItaniumCXXABI::emitDynamicCast(CIRGenFunction &cgf, // if the dynamic type of the pointer is exactly the destination type. if (destRecordTy->getAsCXXRecordDecl()->isEffectivelyFinal() && cgf.cgm.getCodeGenOpts().OptimizationLevel > 0) { - cgm.errorNYI(loc, "emitExactDynamicCast"); - return {}; + CIRGenBuilderTy &builder = cgf.getBuilder(); + // If this isn't a reference cast, check the pointer to see if it's null. + if (!isRefCast) { + mlir::Value srcPtrIsNull = builder.createPtrIsNull(src.getPointer()); + return cir::TernaryOp::create( + builder, loc, srcPtrIsNull, + [&](mlir::OpBuilder, mlir::Location) { + builder.createYield( + loc, builder.getNullPtr(destCIRTy, loc).getResult()); + }, + [&](mlir::OpBuilder &, mlir::Location) { + mlir::Value exactCast = emitExactDynamicCast( + *this, cgf, loc, srcRecordTy, destRecordTy, destCIRTy, + isRefCast, src); + builder.createYield(loc, exactCast); + }) + .getResult(); + } + + return emitExactDynamicCast(*this, cgf, loc, srcRecordTy, destRecordTy, + destCIRTy, isRefCast, src); } cir::DynamicCastInfoAttr castInfo = diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp index 6b29373..46adfe2 100644 --- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp @@ -535,7 +535,7 @@ cir::GlobalOp CIRGenModule::createGlobalOp(CIRGenModule &cgm, builder.setInsertionPointToStart(cgm.getModule().getBody()); } - g = builder.create<cir::GlobalOp>(loc, name, t, isConstant); + g = cir::GlobalOp::create(builder, loc, name, t, isConstant); if (!insertPoint) cgm.lastGlobalOp = g; @@ -739,8 +739,8 @@ mlir::Value CIRGenModule::getAddrOfGlobalVar(const VarDecl *d, mlir::Type ty, cir::GlobalOp g = getOrCreateCIRGlobal(d, ty, isForDefinition); mlir::Type ptrTy = builder.getPointerTo(g.getSymType()); - return builder.create<cir::GetGlobalOp>(getLoc(d->getSourceRange()), ptrTy, - g.getSymName()); + return cir::GetGlobalOp::create(builder, getLoc(d->getSourceRange()), ptrTy, + g.getSymName()); } cir::GlobalViewAttr CIRGenModule::getAddrOfGlobalVarAttr(const VarDecl *d) { @@ -2176,7 +2176,7 @@ CIRGenModule::createCIRFunction(mlir::Location loc, StringRef name, if (cgf) builder.setInsertionPoint(cgf->curFn); - func = builder.create<cir::FuncOp>(loc, name, funcType); + func = cir::FuncOp::create(builder, loc, name, funcType); assert(!cir::MissingFeatures::opFuncAstDeclAttr()); diff --git a/clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp index 5ba6bcb..e7bf3bc 100644 --- a/clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp @@ -27,8 +27,8 @@ mlir::Value createBound(CIRGenFunction &cgf, CIRGen::CIRGenBuilderTy &builder, // Stride is always 1 in C/C++. mlir::Value stride = cgf.createOpenACCConstantInt(boundLoc, 64, 1); - auto bound = - builder.create<mlir::acc::DataBoundsOp>(boundLoc, lowerBound, upperBound); + auto bound = mlir::acc::DataBoundsOp::create(builder, boundLoc, lowerBound, + upperBound); bound.getStartIdxMutable().assign(startIdx); if (extent) bound.getExtentMutable().assign(extent); @@ -48,8 +48,8 @@ mlir::Value CIRGenFunction::emitOpenACCIntExpr(const Expr *intExpr) { ? mlir::IntegerType::SignednessSemantics::Signed : mlir::IntegerType::SignednessSemantics::Unsigned); - auto conversionOp = builder.create<mlir::UnrealizedConversionCastOp>( - exprLoc, targetType, expr); + auto conversionOp = mlir::UnrealizedConversionCastOp::create( + builder, exprLoc, targetType, expr); return conversionOp.getResult(0); } @@ -59,8 +59,8 @@ mlir::Value CIRGenFunction::createOpenACCConstantInt(mlir::Location loc, mlir::IntegerType ty = mlir::IntegerType::get(&getMLIRContext(), width, mlir::IntegerType::SignednessSemantics::Signless); - auto constOp = builder.create<mlir::arith::ConstantOp>( - loc, builder.getIntegerAttr(ty, value)); + auto constOp = mlir::arith::ConstantOp::create( + builder, loc, builder.getIntegerAttr(ty, value)); return constOp; } diff --git a/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp b/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp index 385f89c..5010137 100644 --- a/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp @@ -96,8 +96,8 @@ class OpenACCClauseCIREmitter final mlir::IntegerType targetType = mlir::IntegerType::get( &cgf.getMLIRContext(), /*width=*/1, mlir::IntegerType::SignednessSemantics::Signless); - auto conversionOp = builder.create<mlir::UnrealizedConversionCastOp>( - exprLoc, targetType, condition); + auto conversionOp = mlir::UnrealizedConversionCastOp::create( + builder, exprLoc, targetType, condition); return conversionOp.getResult(0); } @@ -107,8 +107,8 @@ class OpenACCClauseCIREmitter final mlir::IntegerType ty = mlir::IntegerType::get( &cgf.getMLIRContext(), width, mlir::IntegerType::SignednessSemantics::Signless); - auto constOp = builder.create<mlir::arith::ConstantOp>( - loc, builder.getIntegerAttr(ty, value)); + auto constOp = mlir::arith::ConstantOp::create( + builder, loc, builder.getIntegerAttr(ty, value)); return constOp; } @@ -217,8 +217,8 @@ class OpenACCClauseCIREmitter final cgf.getOpenACCDataOperandInfo(varOperand); auto beforeOp = - builder.create<BeforeOpTy>(opInfo.beginLoc, opInfo.varValue, structured, - implicit, opInfo.name, opInfo.bounds); + BeforeOpTy::create(builder, opInfo.beginLoc, opInfo.varValue, + structured, implicit, opInfo.name, opInfo.bounds); operation.getDataClauseOperandsMutable().append(beforeOp.getResult()); AfterOpTy afterOp; @@ -231,12 +231,12 @@ class OpenACCClauseCIREmitter final // Detach/Delete ops don't have the variable reference here, so they // take 1 fewer argument to their build function. afterOp = - builder.create<AfterOpTy>(opInfo.beginLoc, beforeOp, structured, - implicit, opInfo.name, opInfo.bounds); + AfterOpTy::create(builder, opInfo.beginLoc, beforeOp, structured, + implicit, opInfo.name, opInfo.bounds); } else { - afterOp = builder.create<AfterOpTy>( - opInfo.beginLoc, beforeOp, opInfo.varValue, structured, implicit, - opInfo.name, opInfo.bounds); + afterOp = AfterOpTy::create(builder, opInfo.beginLoc, beforeOp, + opInfo.varValue, structured, implicit, + opInfo.name, opInfo.bounds); } } @@ -258,8 +258,8 @@ class OpenACCClauseCIREmitter final CIRGenFunction::OpenACCDataOperandInfo opInfo = cgf.getOpenACCDataOperandInfo(varOperand); auto beforeOp = - builder.create<BeforeOpTy>(opInfo.beginLoc, opInfo.varValue, structured, - implicit, opInfo.name, opInfo.bounds); + BeforeOpTy::create(builder, opInfo.beginLoc, opInfo.varValue, + structured, implicit, opInfo.name, opInfo.bounds); operation.getDataClauseOperandsMutable().append(beforeOp.getResult()); // Set the 'rest' of the info for the operation. diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp index f486c46..1eb7199 100644 --- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp @@ -91,8 +91,9 @@ mlir::LogicalResult CIRGenFunction::emitCompoundStmt(const CompoundStmt &s, SymTableScopeTy varScope(symbolTable); mlir::Location scopeLoc = getLoc(s.getSourceRange()); mlir::OpBuilder::InsertPoint scopeInsPt; - builder.create<cir::ScopeOp>( - scopeLoc, [&](mlir::OpBuilder &b, mlir::Type &type, mlir::Location loc) { + cir::ScopeOp::create( + builder, scopeLoc, + [&](mlir::OpBuilder &b, mlir::Type &type, mlir::Location loc) { scopeInsPt = b.saveInsertionPoint(); }); mlir::OpBuilder::InsertionGuard guard(builder); @@ -423,12 +424,12 @@ mlir::LogicalResult CIRGenFunction::emitIfStmt(const IfStmt &s) { // LexicalScope ConditionScope(*this, S.getCond()->getSourceRange()); // The if scope contains the full source range for IfStmt. mlir::Location scopeLoc = getLoc(s.getSourceRange()); - builder.create<cir::ScopeOp>( - scopeLoc, /*scopeBuilder=*/ - [&](mlir::OpBuilder &b, mlir::Location loc) { - LexicalScope lexScope{*this, scopeLoc, builder.getInsertionBlock()}; - res = ifStmtBuilder(); - }); + cir::ScopeOp::create(builder, scopeLoc, /*scopeBuilder=*/ + [&](mlir::OpBuilder &b, mlir::Location loc) { + LexicalScope lexScope{*this, scopeLoc, + builder.getInsertionBlock()}; + res = ifStmtBuilder(); + }); return res; } @@ -576,11 +577,11 @@ mlir::LogicalResult CIRGenFunction::emitLabel(const clang::LabelDecl &d) { mlir::OpBuilder::InsertionGuard guard(builder); labelBlock = builder.createBlock(builder.getBlock()->getParent()); } - builder.create<cir::BrOp>(getLoc(d.getSourceRange()), labelBlock); + cir::BrOp::create(builder, getLoc(d.getSourceRange()), labelBlock); } builder.setInsertionPointToEnd(labelBlock); - builder.create<cir::LabelOp>(getLoc(d.getSourceRange()), d.getName()); + cir::LabelOp::create(builder, getLoc(d.getSourceRange()), d.getName()); builder.setInsertionPointToEnd(labelBlock); // FIXME: emit debug info for labels, incrementProfileCounter @@ -617,7 +618,7 @@ CIRGenFunction::emitCaseDefaultCascade(const T *stmt, mlir::Type condType, const Stmt *sub = stmt->getSubStmt(); mlir::OpBuilder::InsertPoint insertPoint; - builder.create<CaseOp>(loc, value, kind, insertPoint); + CaseOp::create(builder, loc, value, kind, insertPoint); { mlir::OpBuilder::InsertionGuard guardSwitch(builder); @@ -789,16 +790,16 @@ CIRGenFunction::emitCXXForRangeStmt(const CXXForRangeStmt &s, mlir::LogicalResult res = mlir::success(); mlir::Location scopeLoc = getLoc(s.getSourceRange()); - builder.create<cir::ScopeOp>(scopeLoc, /*scopeBuilder=*/ - [&](mlir::OpBuilder &b, mlir::Location loc) { - // Create a cleanup scope for the condition - // variable cleanups. Logical equivalent from - // LLVM codegn for LexicalScope - // ConditionScope(*this, S.getSourceRange())... - LexicalScope lexScope{ - *this, loc, builder.getInsertionBlock()}; - res = forStmtBuilder(); - }); + cir::ScopeOp::create(builder, scopeLoc, /*scopeBuilder=*/ + [&](mlir::OpBuilder &b, mlir::Location loc) { + // Create a cleanup scope for the condition + // variable cleanups. Logical equivalent from + // LLVM codegn for LexicalScope + // ConditionScope(*this, S.getSourceRange())... + LexicalScope lexScope{*this, loc, + builder.getInsertionBlock()}; + res = forStmtBuilder(); + }); if (res.failed()) return res; @@ -841,7 +842,7 @@ mlir::LogicalResult CIRGenFunction::emitForStmt(const ForStmt &s) { // scalar type. condVal = evaluateExprAsBool(s.getCond()); } else { - condVal = b.create<cir::ConstantOp>(loc, builder.getTrueAttr()); + condVal = cir::ConstantOp::create(b, loc, builder.getTrueAttr()); } builder.createCondition(condVal); }, @@ -865,12 +866,12 @@ mlir::LogicalResult CIRGenFunction::emitForStmt(const ForStmt &s) { auto res = mlir::success(); auto scopeLoc = getLoc(s.getSourceRange()); - builder.create<cir::ScopeOp>(scopeLoc, /*scopeBuilder=*/ - [&](mlir::OpBuilder &b, mlir::Location loc) { - LexicalScope lexScope{ - *this, loc, builder.getInsertionBlock()}; - res = forStmtBuilder(); - }); + cir::ScopeOp::create(builder, scopeLoc, /*scopeBuilder=*/ + [&](mlir::OpBuilder &b, mlir::Location loc) { + LexicalScope lexScope{*this, loc, + builder.getInsertionBlock()}; + res = forStmtBuilder(); + }); if (res.failed()) return res; @@ -916,12 +917,12 @@ mlir::LogicalResult CIRGenFunction::emitDoStmt(const DoStmt &s) { mlir::LogicalResult res = mlir::success(); mlir::Location scopeLoc = getLoc(s.getSourceRange()); - builder.create<cir::ScopeOp>(scopeLoc, /*scopeBuilder=*/ - [&](mlir::OpBuilder &b, mlir::Location loc) { - LexicalScope lexScope{ - *this, loc, builder.getInsertionBlock()}; - res = doStmtBuilder(); - }); + cir::ScopeOp::create(builder, scopeLoc, /*scopeBuilder=*/ + [&](mlir::OpBuilder &b, mlir::Location loc) { + LexicalScope lexScope{*this, loc, + builder.getInsertionBlock()}; + res = doStmtBuilder(); + }); if (res.failed()) return res; @@ -972,12 +973,12 @@ mlir::LogicalResult CIRGenFunction::emitWhileStmt(const WhileStmt &s) { mlir::LogicalResult res = mlir::success(); mlir::Location scopeLoc = getLoc(s.getSourceRange()); - builder.create<cir::ScopeOp>(scopeLoc, /*scopeBuilder=*/ - [&](mlir::OpBuilder &b, mlir::Location loc) { - LexicalScope lexScope{ - *this, loc, builder.getInsertionBlock()}; - res = whileStmtBuilder(); - }); + cir::ScopeOp::create(builder, scopeLoc, /*scopeBuilder=*/ + [&](mlir::OpBuilder &b, mlir::Location loc) { + LexicalScope lexScope{*this, loc, + builder.getInsertionBlock()}; + res = whileStmtBuilder(); + }); if (res.failed()) return res; @@ -1048,8 +1049,8 @@ mlir::LogicalResult CIRGenFunction::emitSwitchStmt(const clang::SwitchStmt &s) { assert(!cir::MissingFeatures::insertBuiltinUnpredictable()); mlir::LogicalResult res = mlir::success(); - swop = builder.create<SwitchOp>( - getLoc(s.getBeginLoc()), condV, + swop = SwitchOp::create( + builder, getLoc(s.getBeginLoc()), condV, /*switchBuilder=*/ [&](mlir::OpBuilder &b, mlir::Location loc, mlir::OperationState &os) { curLexScope->setAsSwitch(); @@ -1067,12 +1068,12 @@ mlir::LogicalResult CIRGenFunction::emitSwitchStmt(const clang::SwitchStmt &s) { // The switch scope contains the full source range for SwitchStmt. mlir::Location scopeLoc = getLoc(s.getSourceRange()); mlir::LogicalResult res = mlir::success(); - builder.create<cir::ScopeOp>(scopeLoc, /*scopeBuilder=*/ - [&](mlir::OpBuilder &b, mlir::Location loc) { - LexicalScope lexScope{ - *this, loc, builder.getInsertionBlock()}; - res = switchStmtBuilder(); - }); + cir::ScopeOp::create(builder, scopeLoc, /*scopeBuilder=*/ + [&](mlir::OpBuilder &b, mlir::Location loc) { + LexicalScope lexScope{*this, loc, + builder.getInsertionBlock()}; + res = switchStmtBuilder(); + }); llvm::SmallVector<CaseOp> cases; swop.collectCases(cases); @@ -1096,7 +1097,7 @@ void CIRGenFunction::emitReturnOfRValue(mlir::Location loc, RValue rv, } mlir::Block *retBlock = curLexScope->getOrCreateRetBlock(*this, loc); assert(!cir::MissingFeatures::emitBranchThroughCleanup()); - builder.create<cir::BrOp>(loc, retBlock); + cir::BrOp::create(builder, loc, retBlock); if (ehStack.stable_begin() != currentCleanupStackDepth) cgm.errorNYI(loc, "return of r-value with cleanup stack"); } diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp index 02bb46d..77e6f83 100644 --- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp @@ -30,7 +30,7 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCOpAssociatedStmt( llvm::SmallVector<mlir::Type> retTy; llvm::SmallVector<mlir::Value> operands; - auto op = builder.create<Op>(start, retTy, operands); + auto op = Op::create(builder, start, retTy, operands); emitOpenACCClauses(op, dirKind, dirLoc, clauses); @@ -42,7 +42,7 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCOpAssociatedStmt( LexicalScope ls{*this, start, builder.getInsertionBlock()}; res = emitStmt(associatedStmt, /*useCurrentScope=*/true); - builder.create<TermOp>(end); + TermOp::create(builder, end); } return res; } @@ -73,7 +73,7 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCOpCombinedConstruct( llvm::SmallVector<mlir::Type> retTy; llvm::SmallVector<mlir::Value> operands; - auto computeOp = builder.create<Op>(start, retTy, operands); + auto computeOp = Op::create(builder, start, retTy, operands); computeOp.setCombinedAttr(builder.getUnitAttr()); mlir::acc::LoopOp loopOp; @@ -85,7 +85,7 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCOpCombinedConstruct( builder.setInsertionPointToEnd(&block); LexicalScope ls{*this, start, builder.getInsertionBlock()}; - auto loopOp = builder.create<LoopOp>(start, retTy, operands); + auto loopOp = LoopOp::create(builder, start, retTy, operands); loopOp.setCombinedAttr(mlir::acc::CombinedConstructsTypeAttr::get( builder.getContext(), CombinedType<Op>::value)); @@ -99,14 +99,14 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCOpCombinedConstruct( res = emitStmt(loopStmt, /*useCurrentScope=*/true); - builder.create<mlir::acc::YieldOp>(end); + mlir::acc::YieldOp::create(builder, end); } emitOpenACCClauses(computeOp, loopOp, dirKind, dirLoc, clauses); updateLoopOpParallelism(loopOp, /*isOrphan=*/false, dirKind); - builder.create<TermOp>(end); + TermOp::create(builder, end); } return res; @@ -118,7 +118,7 @@ Op CIRGenFunction::emitOpenACCOp( llvm::ArrayRef<const OpenACCClause *> clauses) { llvm::SmallVector<mlir::Type> retTy; llvm::SmallVector<mlir::Value> operands; - auto op = builder.create<Op>(start, retTy, operands); + auto op = Op::create(builder, start, retTy, operands); emitOpenACCClauses(op, dirKind, dirLoc, clauses); return op; @@ -197,8 +197,8 @@ CIRGenFunction::emitOpenACCWaitConstruct(const OpenACCWaitConstruct &s) { ? mlir::IntegerType::SignednessSemantics::Signed : mlir::IntegerType::SignednessSemantics::Unsigned); - auto conversionOp = builder.create<mlir::UnrealizedConversionCastOp>( - exprLoc, targetType, expr); + auto conversionOp = mlir::UnrealizedConversionCastOp::create( + builder, exprLoc, targetType, expr); return conversionOp.getResult(0); }; @@ -294,9 +294,9 @@ CIRGenFunction::emitOpenACCCacheConstruct(const OpenACCCacheConstruct &s) { CIRGenFunction::OpenACCDataOperandInfo opInfo = getOpenACCDataOperandInfo(var); - auto cacheOp = builder.create<CacheOp>( - opInfo.beginLoc, opInfo.varValue, - /*structured=*/false, /*implicit=*/false, opInfo.name, opInfo.bounds); + auto cacheOp = CacheOp::create(builder, opInfo.beginLoc, opInfo.varValue, + /*structured=*/false, /*implicit=*/false, + opInfo.name, opInfo.bounds); loopOp.getCacheOperandsMutable().append(cacheOp.getResult()); } diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp index f3911ae..c5b89bd 100644 --- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp @@ -58,7 +58,7 @@ CIRGenFunction::emitOpenACCLoopConstruct(const OpenACCLoopConstruct &s) { mlir::Location end = getLoc(s.getSourceRange().getEnd()); llvm::SmallVector<mlir::Type> retTy; llvm::SmallVector<mlir::Value> operands; - auto op = builder.create<LoopOp>(start, retTy, operands); + auto op = LoopOp::create(builder, start, retTy, operands); // TODO(OpenACC): In the future we are going to need to come up with a // transformation here that can teach the acc.loop how to figure out the @@ -133,7 +133,7 @@ CIRGenFunction::emitOpenACCLoopConstruct(const OpenACCLoopConstruct &s) { ActiveOpenACCLoopRAII activeLoop{*this, &op}; stmtRes = emitStmt(s.getLoop(), /*useCurrentScope=*/true); - builder.create<mlir::acc::YieldOp>(end); + mlir::acc::YieldOp::create(builder, end); } return stmtRes; diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp index fa180f5..2d2ef42 100644 --- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp +++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp @@ -95,8 +95,8 @@ Operation *cir::CIRDialect::materializeConstant(mlir::OpBuilder &builder, mlir::Attribute value, mlir::Type type, mlir::Location loc) { - return builder.create<cir::ConstantOp>(loc, type, - mlir::cast<mlir::TypedAttr>(value)); + return cir::ConstantOp::create(builder, loc, type, + mlir::cast<mlir::TypedAttr>(value)); } //===----------------------------------------------------------------------===// @@ -184,7 +184,7 @@ static LogicalResult ensureRegionTerm(OpAsmParser &parser, Region ®ion, // Terminator was omitted correctly: recreate it. builder.setInsertionPointToEnd(&block); - builder.create<cir::YieldOp>(eLoc); + cir::YieldOp::create(builder, eLoc); return success(); } @@ -977,7 +977,7 @@ void cir::IfOp::print(OpAsmPrinter &p) { /// Default callback for IfOp builders. void cir::buildTerminatedBody(OpBuilder &builder, Location loc) { // add cir.yield to end of the block - builder.create<cir::YieldOp>(loc); + cir::YieldOp::create(builder, loc); } /// Given the region at `index`, or the parent operation if `index` is None, diff --git a/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp b/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp index 7e96ae9..66469e2 100644 --- a/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp +++ b/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp @@ -34,8 +34,8 @@ llvm::SmallVector<MemorySlot> cir::AllocaOp::getPromotableSlots() { Value cir::AllocaOp::getDefaultValue(const MemorySlot &slot, OpBuilder &builder) { - return builder.create<cir::ConstantOp>(getLoc(), - cir::UndefAttr::get(slot.elemType)); + return cir::ConstantOp::create(builder, getLoc(), + cir::UndefAttr::get(slot.elemType)); } void cir::AllocaOp::handleBlockArgument(const MemorySlot &slot, diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp index 46bd186..21c96fe 100644 --- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp +++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp @@ -100,8 +100,8 @@ struct CIRIfFlattening : public mlir::OpRewritePattern<cir::IfOp> { } rewriter.setInsertionPointToEnd(currentBlock); - rewriter.create<cir::BrCondOp>(loc, ifOp.getCondition(), thenBeforeBody, - elseBeforeBody); + cir::BrCondOp::create(rewriter, loc, ifOp.getCondition(), thenBeforeBody, + elseBeforeBody); if (!emptyElse) { rewriter.setInsertionPointToEnd(elseAfterBody); @@ -154,7 +154,7 @@ public: // Save stack and then branch into the body of the region. rewriter.setInsertionPointToEnd(currentBlock); assert(!cir::MissingFeatures::stackSaveOp()); - rewriter.create<cir::BrOp>(loc, mlir::ValueRange(), beforeBody); + cir::BrOp::create(rewriter, loc, mlir::ValueRange(), beforeBody); // Replace the scopeop return with a branch that jumps out of the body. // Stack restore before leaving the body region. @@ -195,26 +195,27 @@ public: cir::IntType sIntType = cir::IntType::get(op.getContext(), 32, true); cir::IntType uIntType = cir::IntType::get(op.getContext(), 32, false); - cir::ConstantOp rangeLength = rewriter.create<cir::ConstantOp>( - op.getLoc(), cir::IntAttr::get(sIntType, upperBound - lowerBound)); + cir::ConstantOp rangeLength = cir::ConstantOp::create( + rewriter, op.getLoc(), + cir::IntAttr::get(sIntType, upperBound - lowerBound)); - cir::ConstantOp lowerBoundValue = rewriter.create<cir::ConstantOp>( - op.getLoc(), cir::IntAttr::get(sIntType, lowerBound)); + cir::ConstantOp lowerBoundValue = cir::ConstantOp::create( + rewriter, op.getLoc(), cir::IntAttr::get(sIntType, lowerBound)); cir::BinOp diffValue = - rewriter.create<cir::BinOp>(op.getLoc(), sIntType, cir::BinOpKind::Sub, - op.getCondition(), lowerBoundValue); + cir::BinOp::create(rewriter, op.getLoc(), sIntType, cir::BinOpKind::Sub, + op.getCondition(), lowerBoundValue); // Use unsigned comparison to check if the condition is in the range. - cir::CastOp uDiffValue = rewriter.create<cir::CastOp>( - op.getLoc(), uIntType, CastKind::integral, diffValue); - cir::CastOp uRangeLength = rewriter.create<cir::CastOp>( - op.getLoc(), uIntType, CastKind::integral, rangeLength); - - cir::CmpOp cmpResult = rewriter.create<cir::CmpOp>( - op.getLoc(), cir::BoolType::get(op.getContext()), cir::CmpOpKind::le, - uDiffValue, uRangeLength); - rewriter.create<cir::BrCondOp>(op.getLoc(), cmpResult, rangeDestination, - defaultDestination); + cir::CastOp uDiffValue = cir::CastOp::create( + rewriter, op.getLoc(), uIntType, CastKind::integral, diffValue); + cir::CastOp uRangeLength = cir::CastOp::create( + rewriter, op.getLoc(), uIntType, CastKind::integral, rangeLength); + + cir::CmpOp cmpResult = cir::CmpOp::create( + rewriter, op.getLoc(), cir::BoolType::get(op.getContext()), + cir::CmpOpKind::le, uDiffValue, uRangeLength); + cir::BrCondOp::create(rewriter, op.getLoc(), cmpResult, rangeDestination, + defaultDestination); return resBlock; } @@ -262,7 +263,7 @@ public: rewriteYieldOp(rewriter, switchYield, exitBlock); rewriter.setInsertionPointToEnd(originalBlock); - rewriter.create<cir::BrOp>(op.getLoc(), swopBlock); + cir::BrOp::create(rewriter, op.getLoc(), swopBlock); } // Allocate required data structures (disconsider default case in @@ -331,8 +332,8 @@ public: mlir::Block *newBlock = rewriter.splitBlock(oldBlock, nextOp->getIterator()); rewriter.setInsertionPointToEnd(oldBlock); - rewriter.create<cir::BrOp>(nextOp->getLoc(), mlir::ValueRange(), - newBlock); + cir::BrOp::create(rewriter, nextOp->getLoc(), mlir::ValueRange(), + newBlock); rewriteYieldOp(rewriter, yieldOp, newBlock); } } @@ -346,7 +347,7 @@ public: // Create a branch to the entry of the inlined region. rewriter.setInsertionPointToEnd(oldBlock); - rewriter.create<cir::BrOp>(caseOp.getLoc(), &entryBlock); + cir::BrOp::create(rewriter, caseOp.getLoc(), &entryBlock); } // Remove all cases since we've inlined the regions. @@ -427,7 +428,7 @@ public: // Setup loop entry branch. rewriter.setInsertionPointToEnd(entry); - rewriter.create<cir::BrOp>(op.getLoc(), &op.getEntry().front()); + cir::BrOp::create(rewriter, op.getLoc(), &op.getEntry().front()); // Branch from condition region to body or exit. auto conditionOp = cast<cir::ConditionOp>(cond->getTerminator()); @@ -499,7 +500,7 @@ public: locs.push_back(loc); Block *continueBlock = rewriter.createBlock(remainingOpsBlock, op->getResultTypes(), locs); - rewriter.create<cir::BrOp>(loc, remainingOpsBlock); + cir::BrOp::create(rewriter, loc, remainingOpsBlock); Region &trueRegion = op.getTrueRegion(); Block *trueBlock = &trueRegion.front(); @@ -542,7 +543,7 @@ public: rewriter.inlineRegionBefore(falseRegion, continueBlock); rewriter.setInsertionPointToEnd(condBlock); - rewriter.create<cir::BrCondOp>(loc, op.getCond(), trueBlock, falseBlock); + cir::BrCondOp::create(rewriter, loc, op.getCond(), trueBlock, falseBlock); rewriter.replaceOp(op, continueBlock->getArguments()); diff --git a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp index d99c362..cba0464 100644 --- a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp +++ b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp @@ -155,7 +155,7 @@ cir::FuncOp LoweringPreparePass::buildRuntimeFunction( cir::FuncOp f = dyn_cast_or_null<FuncOp>(SymbolTable::lookupNearestSymbolFrom( mlirModule, StringAttr::get(mlirModule->getContext(), name))); if (!f) { - f = builder.create<cir::FuncOp>(loc, name, type); + f = cir::FuncOp::create(builder, loc, name, type); f.setLinkageAttr( cir::GlobalLinkageKindAttr::get(builder.getContext(), linkage)); mlir::SymbolTable::setSymbolVisibility( @@ -400,12 +400,12 @@ buildRangeReductionComplexDiv(CIRBaseBuilderTy &builder, mlir::Location loc, builder.createYield(loc, result); }; - auto cFabs = builder.create<cir::FAbsOp>(loc, c); - auto dFabs = builder.create<cir::FAbsOp>(loc, d); + auto cFabs = cir::FAbsOp::create(builder, loc, c); + auto dFabs = cir::FAbsOp::create(builder, loc, d); cir::CmpOp cmpResult = builder.createCompare(loc, cir::CmpOpKind::ge, cFabs, dFabs); - auto ternary = builder.create<cir::TernaryOp>( - loc, cmpResult, trueBranchBuilder, falseBranchBuilder); + auto ternary = cir::TernaryOp::create(builder, loc, cmpResult, + trueBranchBuilder, falseBranchBuilder); return ternary.getResult(); } @@ -612,18 +612,17 @@ static mlir::Value lowerComplexMul(LoweringPreparePass &pass, mlir::Value resultRealAndImagAreNaN = builder.createLogicalAnd(loc, resultRealIsNaN, resultImagIsNaN); - return builder - .create<cir::TernaryOp>( - loc, resultRealAndImagAreNaN, - [&](mlir::OpBuilder &, mlir::Location) { - mlir::Value libCallResult = buildComplexBinOpLibCall( - pass, builder, &getComplexMulLibCallName, loc, complexTy, - lhsReal, lhsImag, rhsReal, rhsImag); - builder.createYield(loc, libCallResult); - }, - [&](mlir::OpBuilder &, mlir::Location) { - builder.createYield(loc, algebraicResult); - }) + return cir::TernaryOp::create( + builder, loc, resultRealAndImagAreNaN, + [&](mlir::OpBuilder &, mlir::Location) { + mlir::Value libCallResult = buildComplexBinOpLibCall( + pass, builder, &getComplexMulLibCallName, loc, complexTy, + lhsReal, lhsImag, rhsReal, rhsImag); + builder.createYield(loc, libCallResult); + }, + [&](mlir::OpBuilder &, mlir::Location) { + builder.createYield(loc, algebraicResult); + }) .getResult(); } @@ -920,15 +919,15 @@ static void lowerArrayDtorCtorIntoLoop(cir::CIRBaseBuilderTy &builder, loc, /*condBuilder=*/ [&](mlir::OpBuilder &b, mlir::Location loc) { - auto currentElement = b.create<cir::LoadOp>(loc, eltTy, tmpAddr); + auto currentElement = cir::LoadOp::create(b, loc, eltTy, tmpAddr); mlir::Type boolTy = cir::BoolType::get(b.getContext()); - auto cmp = builder.create<cir::CmpOp>(loc, boolTy, cir::CmpOpKind::ne, - currentElement, stop); + auto cmp = cir::CmpOp::create(builder, loc, boolTy, cir::CmpOpKind::ne, + currentElement, stop); builder.createCondition(cmp); }, /*bodyBuilder=*/ [&](mlir::OpBuilder &b, mlir::Location loc) { - auto currentElement = b.create<cir::LoadOp>(loc, eltTy, tmpAddr); + auto currentElement = cir::LoadOp::create(b, loc, eltTy, tmpAddr); cir::CallOp ctorCall; op->walk([&](cir::CallOp c) { ctorCall = c; }); diff --git a/clang/lib/CIR/Dialect/Transforms/LoweringPrepareItaniumCXXABI.cpp b/clang/lib/CIR/Dialect/Transforms/LoweringPrepareItaniumCXXABI.cpp index 11ce2a8..5a067f8 100644 --- a/clang/lib/CIR/Dialect/Transforms/LoweringPrepareItaniumCXXABI.cpp +++ b/clang/lib/CIR/Dialect/Transforms/LoweringPrepareItaniumCXXABI.cpp @@ -77,10 +77,11 @@ buildDynamicCastAfterNullCheck(cir::CIRBaseBuilderTy &builder, if (op.isRefCast()) { // Emit a cir.if that checks the casted value. mlir::Value castedValueIsNull = builder.createPtrIsNull(castedPtr); - builder.create<cir::IfOp>( - loc, castedValueIsNull, false, [&](mlir::OpBuilder &, mlir::Location) { - buildBadCastCall(builder, loc, castInfo.getBadCastFunc()); - }); + cir::IfOp::create(builder, loc, castedValueIsNull, false, + [&](mlir::OpBuilder &, mlir::Location) { + buildBadCastCall(builder, loc, + castInfo.getBadCastFunc()); + }); } // Note that castedPtr is a void*. Cast it to a pointer to the destination @@ -154,19 +155,19 @@ LoweringPrepareItaniumCXXABI::lowerDynamicCast(cir::CIRBaseBuilderTy &builder, return buildDynamicCastAfterNullCheck(builder, op); mlir::Value srcValueIsNotNull = builder.createPtrToBoolCast(srcValue); - return builder - .create<cir::TernaryOp>( - loc, srcValueIsNotNull, - [&](mlir::OpBuilder &, mlir::Location) { - mlir::Value castedValue = - op.isCastToVoid() - ? buildDynamicCastToVoidAfterNullCheck(builder, astCtx, op) - : buildDynamicCastAfterNullCheck(builder, op); - builder.createYield(loc, castedValue); - }, - [&](mlir::OpBuilder &, mlir::Location) { - builder.createYield( - loc, builder.getNullPtr(op.getType(), loc).getResult()); - }) + return cir::TernaryOp::create( + builder, loc, srcValueIsNotNull, + [&](mlir::OpBuilder &, mlir::Location) { + mlir::Value castedValue = + op.isCastToVoid() + ? buildDynamicCastToVoidAfterNullCheck(builder, astCtx, + op) + : buildDynamicCastAfterNullCheck(builder, op); + builder.createYield(loc, castedValue); + }, + [&](mlir::OpBuilder &, mlir::Location) { + builder.createYield( + loc, builder.getNullPtr(op.getType(), loc).getResult()); + }) .getResult(); } diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index bb75f2d..a30ae02 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -90,12 +90,12 @@ static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src, mlir::Location loc = src.getLoc(); if (dstWidth > srcWidth && isSigned) - return bld.create<mlir::LLVM::SExtOp>(loc, dstTy, src); + return mlir::LLVM::SExtOp::create(bld, loc, dstTy, src); if (dstWidth > srcWidth) - return bld.create<mlir::LLVM::ZExtOp>(loc, dstTy, src); + return mlir::LLVM::ZExtOp::create(bld, loc, dstTy, src); if (dstWidth < srcWidth) - return bld.create<mlir::LLVM::TruncOp>(loc, dstTy, src); - return bld.create<mlir::LLVM::BitcastOp>(loc, dstTy, src); + return mlir::LLVM::TruncOp::create(bld, loc, dstTy, src); + return mlir::LLVM::BitcastOp::create(bld, loc, dstTy, src); } static mlir::LLVM::Visibility @@ -204,12 +204,12 @@ static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter, auto loc = llvmSrc.getLoc(); if (cirSrcWidth < cirDstIntWidth) { if (isUnsigned) - return rewriter.create<mlir::LLVM::ZExtOp>(loc, llvmDstIntTy, llvmSrc); - return rewriter.create<mlir::LLVM::SExtOp>(loc, llvmDstIntTy, llvmSrc); + return mlir::LLVM::ZExtOp::create(rewriter, loc, llvmDstIntTy, llvmSrc); + return mlir::LLVM::SExtOp::create(rewriter, loc, llvmDstIntTy, llvmSrc); } // Otherwise truncate - return rewriter.create<mlir::LLVM::TruncOp>(loc, llvmDstIntTy, llvmSrc); + return mlir::LLVM::TruncOp::create(rewriter, loc, llvmDstIntTy, llvmSrc); } class CIRAttrToValue { @@ -315,15 +315,17 @@ static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp( /// IntAttr visitor. mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) { mlir::Location loc = parentOp->getLoc(); - return rewriter.create<mlir::LLVM::ConstantOp>( - loc, converter->convertType(intAttr.getType()), intAttr.getValue()); + return mlir::LLVM::ConstantOp::create( + rewriter, loc, converter->convertType(intAttr.getType()), + intAttr.getValue()); } /// FPAttr visitor. mlir::Value CIRAttrToValue::visitCirAttr(cir::FPAttr fltAttr) { mlir::Location loc = parentOp->getLoc(); - return rewriter.create<mlir::LLVM::ConstantOp>( - loc, converter->convertType(fltAttr.getType()), fltAttr.getValue()); + return mlir::LLVM::ConstantOp::create( + rewriter, loc, converter->convertType(fltAttr.getType()), + fltAttr.getValue()); } /// ConstComplexAttr visitor. @@ -350,8 +352,8 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstComplexAttr complexAttr) { } mlir::Location loc = parentOp->getLoc(); - return rewriter.create<mlir::LLVM::ConstantOp>( - loc, converter->convertType(complexAttr.getType()), + return mlir::LLVM::ConstantOp::create( + rewriter, loc, converter->convertType(complexAttr.getType()), rewriter.getArrayAttr(components)); } @@ -359,15 +361,16 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstComplexAttr complexAttr) { mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstPtrAttr ptrAttr) { mlir::Location loc = parentOp->getLoc(); if (ptrAttr.isNullValue()) { - return rewriter.create<mlir::LLVM::ZeroOp>( - loc, converter->convertType(ptrAttr.getType())); + return mlir::LLVM::ZeroOp::create( + rewriter, loc, converter->convertType(ptrAttr.getType())); } mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>()); - mlir::Value ptrVal = rewriter.create<mlir::LLVM::ConstantOp>( - loc, rewriter.getIntegerType(layout.getTypeSizeInBits(ptrAttr.getType())), + mlir::Value ptrVal = mlir::LLVM::ConstantOp::create( + rewriter, loc, + rewriter.getIntegerType(layout.getTypeSizeInBits(ptrAttr.getType())), ptrAttr.getValue().getInt()); - return rewriter.create<mlir::LLVM::IntToPtrOp>( - loc, converter->convertType(ptrAttr.getType()), ptrVal); + return mlir::LLVM::IntToPtrOp::create( + rewriter, loc, converter->convertType(ptrAttr.getType()), ptrVal); } // ConstArrayAttr visitor @@ -378,10 +381,10 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) { if (attr.hasTrailingZeros()) { mlir::Type arrayTy = attr.getType(); - result = rewriter.create<mlir::LLVM::ZeroOp>( - loc, converter->convertType(arrayTy)); + result = mlir::LLVM::ZeroOp::create(rewriter, loc, + converter->convertType(arrayTy)); } else { - result = rewriter.create<mlir::LLVM::UndefOp>(loc, llvmTy); + result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy); } // Iteratively lower each constant element of the array. @@ -390,7 +393,7 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) { mlir::DataLayout dataLayout(parentOp->getParentOfType<mlir::ModuleOp>()); mlir::Value init = visit(elt); result = - rewriter.create<mlir::LLVM::InsertValueOp>(loc, result, init, idx); + mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx); } } else if (auto strAttr = mlir::dyn_cast<mlir::StringAttr>(attr.getElts())) { // TODO(cir): this diverges from traditional lowering. Normally the string @@ -399,10 +402,10 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) { assert(arrayTy && "String attribute must have an array type"); mlir::Type eltTy = arrayTy.getElementType(); for (auto [idx, elt] : llvm::enumerate(strAttr)) { - auto init = rewriter.create<mlir::LLVM::ConstantOp>( - loc, converter->convertType(eltTy), elt); + auto init = mlir::LLVM::ConstantOp::create( + rewriter, loc, converter->convertType(eltTy), elt); result = - rewriter.create<mlir::LLVM::InsertValueOp>(loc, result, init, idx); + mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx); } } else { llvm_unreachable("unexpected ConstArrayAttr elements"); @@ -415,12 +418,13 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) { mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstRecordAttr constRecord) { const mlir::Type llvmTy = converter->convertType(constRecord.getType()); const mlir::Location loc = parentOp->getLoc(); - mlir::Value result = rewriter.create<mlir::LLVM::UndefOp>(loc, llvmTy); + mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy); // Iteratively lower each constant element of the record. for (auto [idx, elt] : llvm::enumerate(constRecord.getMembers())) { mlir::Value init = visit(elt); - result = rewriter.create<mlir::LLVM::InsertValueOp>(loc, result, init, idx); + result = + mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx); } return result; @@ -447,8 +451,8 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstVectorAttr attr) { mlirValues.push_back(mlirAttr); } - return rewriter.create<mlir::LLVM::ConstantOp>( - loc, llvmTy, + return mlir::LLVM::ConstantOp::create( + rewriter, loc, llvmTy, mlir::DenseElementsAttr::get(mlir::cast<mlir::ShapedType>(llvmTy), mlirValues)); } @@ -483,8 +487,9 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::GlobalViewAttr globalAttr) { } mlir::Location loc = parentOp->getLoc(); - mlir::Value addrOp = rewriter.create<mlir::LLVM::AddressOfOp>( - loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), symName); + mlir::Value addrOp = mlir::LLVM::AddressOfOp::create( + rewriter, loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), + symName); if (globalAttr.getIndices()) { llvm::SmallVector<mlir::LLVM::GEPArg> indices; @@ -499,8 +504,9 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::GlobalViewAttr globalAttr) { } mlir::Type resTy = addrOp.getType(); mlir::Type eltTy = converter->convertType(sourceType); - addrOp = rewriter.create<mlir::LLVM::GEPOp>( - loc, resTy, eltTy, addrOp, indices, mlir::LLVM::GEPNoWrapFlags::none); + addrOp = + mlir::LLVM::GEPOp::create(rewriter, loc, resTy, eltTy, addrOp, indices, + mlir::LLVM::GEPNoWrapFlags::none); } // The incubator has handling here for the attribute having integer type, but @@ -517,8 +523,8 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::GlobalViewAttr globalAttr) { return addrOp; mlir::Type llvmDstTy = converter->convertType(globalAttr.getType()); - return rewriter.create<mlir::LLVM::BitcastOp>(parentOp->getLoc(), llvmDstTy, - addrOp); + return mlir::LLVM::BitcastOp::create(rewriter, parentOp->getLoc(), + llvmDstTy, addrOp); } llvm_unreachable("Expecting pointer or integer type for GlobalViewAttr"); @@ -557,8 +563,8 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::VTableAttr vtableArr) { /// ZeroAttr visitor. mlir::Value CIRAttrToValue::visitCirAttr(cir::ZeroAttr attr) { mlir::Location loc = parentOp->getLoc(); - return rewriter.create<mlir::LLVM::ZeroOp>( - loc, converter->convertType(attr.getType())); + return mlir::LLVM::ZeroOp::create(rewriter, loc, + converter->convertType(attr.getType())); } // This class handles rewriting initializer attributes for types that do not @@ -666,8 +672,8 @@ mlir::LogicalResult CIRToLLVMAssumeAlignedOpLowering::matchAndRewrite( mlir::LogicalResult CIRToLLVMAssumeSepStorageOpLowering::matchAndRewrite( cir::AssumeSepStorageOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const { - auto cond = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), - rewriter.getI1Type(), 1); + auto cond = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), + rewriter.getI1Type(), 1); rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>( op, cond, mlir::LLVM::AssumeSeparateStorageTag{}, adaptor.getPtr1(), adaptor.getPtr2()); @@ -914,28 +920,28 @@ mlir::LogicalResult CIRToLLVMAtomicFetchOpLowering::matchAndRewrite( mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite( cir::BitClrsbOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const { - auto zero = rewriter.create<mlir::LLVM::ConstantOp>( - op.getLoc(), adaptor.getInput().getType(), 0); - auto isNeg = rewriter.create<mlir::LLVM::ICmpOp>( - op.getLoc(), + auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), + adaptor.getInput().getType(), 0); + auto isNeg = mlir::LLVM::ICmpOp::create( + rewriter, op.getLoc(), mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(), mlir::LLVM::ICmpPredicate::slt), adaptor.getInput(), zero); - auto negOne = rewriter.create<mlir::LLVM::ConstantOp>( - op.getLoc(), adaptor.getInput().getType(), -1); - auto flipped = rewriter.create<mlir::LLVM::XOrOp>(op.getLoc(), - adaptor.getInput(), negOne); + auto negOne = mlir::LLVM::ConstantOp::create( + rewriter, op.getLoc(), adaptor.getInput().getType(), -1); + auto flipped = mlir::LLVM::XOrOp::create(rewriter, op.getLoc(), + adaptor.getInput(), negOne); - auto select = rewriter.create<mlir::LLVM::SelectOp>( - op.getLoc(), isNeg, flipped, adaptor.getInput()); + auto select = mlir::LLVM::SelectOp::create(rewriter, op.getLoc(), isNeg, + flipped, adaptor.getInput()); auto resTy = getTypeConverter()->convertType(op.getType()); - auto clz = rewriter.create<mlir::LLVM::CountLeadingZerosOp>( - op.getLoc(), resTy, select, /*is_zero_poison=*/false); + auto clz = mlir::LLVM::CountLeadingZerosOp::create( + rewriter, op.getLoc(), resTy, select, /*is_zero_poison=*/false); - auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1); - auto res = rewriter.create<mlir::LLVM::SubOp>(op.getLoc(), clz, one); + auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1); + auto res = mlir::LLVM::SubOp::create(rewriter, op.getLoc(), clz, one); rewriter.replaceOp(op, res); return mlir::LogicalResult::success(); @@ -945,8 +951,8 @@ mlir::LogicalResult CIRToLLVMBitClzOpLowering::matchAndRewrite( cir::BitClzOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const { auto resTy = getTypeConverter()->convertType(op.getType()); - auto llvmOp = rewriter.create<mlir::LLVM::CountLeadingZerosOp>( - op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero()); + auto llvmOp = mlir::LLVM::CountLeadingZerosOp::create( + rewriter, op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero()); rewriter.replaceOp(op, llvmOp); return mlir::LogicalResult::success(); } @@ -955,8 +961,8 @@ mlir::LogicalResult CIRToLLVMBitCtzOpLowering::matchAndRewrite( cir::BitCtzOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const { auto resTy = getTypeConverter()->convertType(op.getType()); - auto llvmOp = rewriter.create<mlir::LLVM::CountTrailingZerosOp>( - op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero()); + auto llvmOp = mlir::LLVM::CountTrailingZerosOp::create( + rewriter, op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero()); rewriter.replaceOp(op, llvmOp); return mlir::LogicalResult::success(); } @@ -965,23 +971,24 @@ mlir::LogicalResult CIRToLLVMBitFfsOpLowering::matchAndRewrite( cir::BitFfsOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const { auto resTy = getTypeConverter()->convertType(op.getType()); - auto ctz = rewriter.create<mlir::LLVM::CountTrailingZerosOp>( - op.getLoc(), resTy, adaptor.getInput(), /*is_zero_poison=*/true); + auto ctz = mlir::LLVM::CountTrailingZerosOp::create(rewriter, op.getLoc(), + resTy, adaptor.getInput(), + /*is_zero_poison=*/true); - auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1); - auto ctzAddOne = rewriter.create<mlir::LLVM::AddOp>(op.getLoc(), ctz, one); + auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1); + auto ctzAddOne = mlir::LLVM::AddOp::create(rewriter, op.getLoc(), ctz, one); - auto zeroInputTy = rewriter.create<mlir::LLVM::ConstantOp>( - op.getLoc(), adaptor.getInput().getType(), 0); - auto isZero = rewriter.create<mlir::LLVM::ICmpOp>( - op.getLoc(), + auto zeroInputTy = mlir::LLVM::ConstantOp::create( + rewriter, op.getLoc(), adaptor.getInput().getType(), 0); + auto isZero = mlir::LLVM::ICmpOp::create( + rewriter, op.getLoc(), mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(), mlir::LLVM::ICmpPredicate::eq), adaptor.getInput(), zeroInputTy); - auto zero = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 0); - auto res = rewriter.create<mlir::LLVM::SelectOp>(op.getLoc(), isZero, zero, - ctzAddOne); + auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 0); + auto res = mlir::LLVM::SelectOp::create(rewriter, op.getLoc(), isZero, zero, + ctzAddOne); rewriter.replaceOp(op, res); return mlir::LogicalResult::success(); @@ -991,12 +998,12 @@ mlir::LogicalResult CIRToLLVMBitParityOpLowering::matchAndRewrite( cir::BitParityOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const { auto resTy = getTypeConverter()->convertType(op.getType()); - auto popcnt = rewriter.create<mlir::LLVM::CtPopOp>(op.getLoc(), resTy, - adaptor.getInput()); + auto popcnt = mlir::LLVM::CtPopOp::create(rewriter, op.getLoc(), resTy, + adaptor.getInput()); - auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1); + auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1); auto popcntMod2 = - rewriter.create<mlir::LLVM::AndOp>(op.getLoc(), popcnt, one); + mlir::LLVM::AndOp::create(rewriter, op.getLoc(), popcnt, one); rewriter.replaceOp(op, popcntMod2); return mlir::LogicalResult::success(); @@ -1006,8 +1013,8 @@ mlir::LogicalResult CIRToLLVMBitPopcountOpLowering::matchAndRewrite( cir::BitPopcountOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const { auto resTy = getTypeConverter()->convertType(op.getType()); - auto llvmOp = rewriter.create<mlir::LLVM::CtPopOp>(op.getLoc(), resTy, - adaptor.getInput()); + auto llvmOp = mlir::LLVM::CtPopOp::create(rewriter, op.getLoc(), resTy, + adaptor.getInput()); rewriter.replaceOp(op, llvmOp); return mlir::LogicalResult::success(); } @@ -1067,8 +1074,8 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite( } case cir::CastKind::int_to_bool: { mlir::Value llvmSrcVal = adaptor.getSrc(); - mlir::Value zeroInt = rewriter.create<mlir::LLVM::ConstantOp>( - castOp.getLoc(), llvmSrcVal.getType(), 0); + mlir::Value zeroInt = mlir::LLVM::ConstantOp::create( + rewriter, castOp.getLoc(), llvmSrcVal.getType(), 0); rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>( castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroInt); break; @@ -1132,8 +1139,8 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite( auto kind = mlir::LLVM::FCmpPredicate::une; // Check if float is not equal to zero. - auto zeroFloat = rewriter.create<mlir::LLVM::ConstantOp>( - castOp.getLoc(), llvmSrcVal.getType(), + auto zeroFloat = mlir::LLVM::ConstantOp::create( + rewriter, castOp.getLoc(), llvmSrcVal.getType(), mlir::FloatAttr::get(llvmSrcVal.getType(), 0.0)); // Extend comparison result to either bool (C++) or int (C). @@ -1204,8 +1211,8 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite( } case cir::CastKind::ptr_to_bool: { mlir::Value llvmSrcVal = adaptor.getSrc(); - mlir::Value zeroPtr = rewriter.create<mlir::LLVM::ZeroOp>( - castOp.getLoc(), llvmSrcVal.getType()); + mlir::Value zeroPtr = mlir::LLVM::ZeroOp::create(rewriter, castOp.getLoc(), + llvmSrcVal.getType()); rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>( castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroPtr); break; @@ -1275,10 +1282,10 @@ mlir::LogicalResult CIRToLLVMPtrStrideOpLowering::matchAndRewrite( // Rewrite the sub in front of extensions/trunc if (rewriteSub) { - index = rewriter.create<mlir::LLVM::SubOp>( - index.getLoc(), index.getType(), - rewriter.create<mlir::LLVM::ConstantOp>(index.getLoc(), - index.getType(), 0), + index = mlir::LLVM::SubOp::create( + rewriter, index.getLoc(), index.getType(), + mlir::LLVM::ConstantOp::create(rewriter, index.getLoc(), + index.getType(), 0), index); rewriter.eraseOp(sub); } @@ -1310,11 +1317,11 @@ mlir::LogicalResult CIRToLLVMBaseClassAddrOpLowering::matchAndRewrite( baseClassOp, resultType, byteType, derivedAddr, offset); } else { auto loc = baseClassOp.getLoc(); - mlir::Value isNull = rewriter.create<mlir::LLVM::ICmpOp>( - loc, mlir::LLVM::ICmpPredicate::eq, derivedAddr, - rewriter.create<mlir::LLVM::ZeroOp>(loc, derivedAddr.getType())); - mlir::Value adjusted = rewriter.create<mlir::LLVM::GEPOp>( - loc, resultType, byteType, derivedAddr, offset); + mlir::Value isNull = mlir::LLVM::ICmpOp::create( + rewriter, loc, mlir::LLVM::ICmpPredicate::eq, derivedAddr, + mlir::LLVM::ZeroOp::create(rewriter, loc, derivedAddr.getType())); + mlir::Value adjusted = mlir::LLVM::GEPOp::create( + rewriter, loc, resultType, byteType, derivedAddr, offset); rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(baseClassOp, isNull, derivedAddr, adjusted); } @@ -1335,8 +1342,8 @@ mlir::LogicalResult CIRToLLVMAllocaOpLowering::matchAndRewrite( mlir::Value size = op.isDynamic() ? adaptor.getDynAllocSize() - : rewriter.create<mlir::LLVM::ConstantOp>( - op.getLoc(), + : mlir::LLVM::ConstantOp::create( + rewriter, op.getLoc(), typeConverter->convertType(rewriter.getIndexType()), 1); mlir::Type elementTy = convertTypeForMemory(*getTypeConverter(), dataLayout, op.getAllocaType()); @@ -1694,13 +1701,13 @@ mlir::LogicalResult CIRToLLVMPtrDiffOpLowering::matchAndRewrite( auto dstTy = mlir::cast<cir::IntType>(op.getType()); mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy); - auto lhs = rewriter.create<mlir::LLVM::PtrToIntOp>(op.getLoc(), llvmDstTy, - adaptor.getLhs()); - auto rhs = rewriter.create<mlir::LLVM::PtrToIntOp>(op.getLoc(), llvmDstTy, - adaptor.getRhs()); + auto lhs = mlir::LLVM::PtrToIntOp::create(rewriter, op.getLoc(), llvmDstTy, + adaptor.getLhs()); + auto rhs = mlir::LLVM::PtrToIntOp::create(rewriter, op.getLoc(), llvmDstTy, + adaptor.getRhs()); auto diff = - rewriter.create<mlir::LLVM::SubOp>(op.getLoc(), llvmDstTy, lhs, rhs); + mlir::LLVM::SubOp::create(rewriter, op.getLoc(), llvmDstTy, lhs, rhs); cir::PointerType ptrTy = op.getLhs().getType(); assert(!cir::MissingFeatures::llvmLoweringPtrDiffConsidersPointee()); @@ -1709,17 +1716,17 @@ mlir::LogicalResult CIRToLLVMPtrDiffOpLowering::matchAndRewrite( // Avoid silly division by 1. mlir::Value resultVal = diff.getResult(); if (typeSize != 1) { - auto typeSizeVal = rewriter.create<mlir::LLVM::ConstantOp>( - op.getLoc(), llvmDstTy, typeSize); + auto typeSizeVal = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), + llvmDstTy, typeSize); if (dstTy.isUnsigned()) { auto uDiv = - rewriter.create<mlir::LLVM::UDivOp>(op.getLoc(), diff, typeSizeVal); + mlir::LLVM::UDivOp::create(rewriter, op.getLoc(), diff, typeSizeVal); uDiv.setIsExact(true); resultVal = uDiv.getResult(); } else { auto sDiv = - rewriter.create<mlir::LLVM::SDivOp>(op.getLoc(), diff, typeSizeVal); + mlir::LLVM::SDivOp::create(rewriter, op.getLoc(), diff, typeSizeVal); sDiv.setIsExact(true); resultVal = sDiv.getResult(); } @@ -1847,8 +1854,8 @@ mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewrite( SmallVector<mlir::NamedAttribute, 4> attributes; lowerFuncAttributes(op, /*filterArgAndResAttrs=*/false, attributes); - mlir::LLVM::LLVMFuncOp fn = rewriter.create<mlir::LLVM::LLVMFuncOp>( - loc, op.getName(), llvmFnTy, linkage, isDsoLocal, cconv, + mlir::LLVM::LLVMFuncOp fn = mlir::LLVM::LLVMFuncOp::create( + rewriter, loc, op.getName(), llvmFnTy, linkage, isDsoLocal, cconv, mlir::SymbolRefAttr(), attributes); assert(!cir::MissingFeatures::opFuncMultipleReturnVals()); @@ -1884,8 +1891,8 @@ mlir::LogicalResult CIRToLLVMGetGlobalOpLowering::matchAndRewrite( } mlir::Type type = getTypeConverter()->convertType(op.getType()); - mlir::Operation *newop = - rewriter.create<mlir::LLVM::AddressOfOp>(op.getLoc(), type, op.getName()); + mlir::Operation *newop = mlir::LLVM::AddressOfOp::create( + rewriter, op.getLoc(), type, op.getName()); assert(!cir::MissingFeatures::opGlobalThreadLocal()); @@ -1941,7 +1948,7 @@ CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal( setupRegionInitializedLLVMGlobalOp(op, rewriter); CIRAttrToValue valueConverter(op, rewriter, typeConverter); mlir::Value value = valueConverter.visit(init); - rewriter.create<mlir::LLVM::ReturnOp>(loc, value); + mlir::LLVM::ReturnOp::create(rewriter, loc, value); return mlir::success(); } @@ -2094,14 +2101,14 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite( switch (op.getKind()) { case cir::UnaryOpKind::Inc: { assert(!isVector && "++ not allowed on vector types"); - auto one = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 1); + auto one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1); rewriter.replaceOpWithNewOp<mlir::LLVM::AddOp>( op, llvmType, adaptor.getInput(), one, maybeNSW); return mlir::success(); } case cir::UnaryOpKind::Dec: { assert(!isVector && "-- not allowed on vector types"); - auto one = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 1); + auto one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1); rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(op, adaptor.getInput(), one, maybeNSW); return mlir::success(); @@ -2112,9 +2119,9 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite( case cir::UnaryOpKind::Minus: { mlir::Value zero; if (isVector) - zero = rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmType); + zero = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmType); else - zero = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 0); + zero = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 0); rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>( op, zero, adaptor.getInput(), maybeNSW); return mlir::success(); @@ -2128,9 +2135,9 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite( std::vector<int32_t> values(numElements, -1); mlir::DenseIntElementsAttr denseVec = rewriter.getI32VectorAttr(values); minusOne = - rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, denseVec); + mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, denseVec); } else { - minusOne = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, -1); + minusOne = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, -1); } rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(), minusOne); @@ -2145,16 +2152,16 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite( switch (op.getKind()) { case cir::UnaryOpKind::Inc: { assert(!isVector && "++ not allowed on vector types"); - mlir::LLVM::ConstantOp one = rewriter.create<mlir::LLVM::ConstantOp>( - loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0)); + mlir::LLVM::ConstantOp one = mlir::LLVM::ConstantOp::create( + rewriter, loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0)); rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, llvmType, one, adaptor.getInput()); return mlir::success(); } case cir::UnaryOpKind::Dec: { assert(!isVector && "-- not allowed on vector types"); - mlir::LLVM::ConstantOp minusOne = rewriter.create<mlir::LLVM::ConstantOp>( - loc, llvmType, rewriter.getFloatAttr(llvmType, -1.0)); + mlir::LLVM::ConstantOp minusOne = mlir::LLVM::ConstantOp::create( + rewriter, loc, llvmType, rewriter.getFloatAttr(llvmType, -1.0)); rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, llvmType, minusOne, adaptor.getInput()); return mlir::success(); @@ -2185,7 +2192,7 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite( return op.emitError() << "Unsupported unary operation on boolean type"; case cir::UnaryOpKind::Not: { assert(!isVector && "NYI: op! on vector mask"); - auto one = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 1); + auto one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1); rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(), one); return mlir::success(); @@ -2404,6 +2411,15 @@ mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite( return mlir::success(); } + if (auto vptrTy = mlir::dyn_cast<cir::VPtrType>(type)) { + // !cir.vptr is a special case, but it's just a pointer to LLVM. + auto kind = convertCmpKindToICmpPredicate(cmpOp.getKind(), + /* isSigned=*/false); + rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>( + cmpOp, kind, adaptor.getLhs(), adaptor.getRhs()); + return mlir::success(); + } + if (mlir::isa<cir::FPTypeInterface>(type)) { mlir::LLVM::FCmpPredicate kind = convertCmpKindToFCmpPredicate(cmpOp.getKind()); @@ -2421,47 +2437,47 @@ mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite( mlir::Type complexElemTy = getTypeConverter()->convertType(complexType.getElementType()); - auto lhsReal = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0); - auto lhsImag = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1); - auto rhsReal = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0); - auto rhsImag = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1); + auto lhsReal = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0})); + auto lhsImag = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1})); + auto rhsReal = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0})); + auto rhsImag = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1})); if (cmpOp.getKind() == cir::CmpOpKind::eq) { if (complexElemTy.isInteger()) { - auto realCmp = rewriter.create<mlir::LLVM::ICmpOp>( - loc, mlir::LLVM::ICmpPredicate::eq, lhsReal, rhsReal); - auto imagCmp = rewriter.create<mlir::LLVM::ICmpOp>( - loc, mlir::LLVM::ICmpPredicate::eq, lhsImag, rhsImag); + auto realCmp = mlir::LLVM::ICmpOp::create( + rewriter, loc, mlir::LLVM::ICmpPredicate::eq, lhsReal, rhsReal); + auto imagCmp = mlir::LLVM::ICmpOp::create( + rewriter, loc, mlir::LLVM::ICmpPredicate::eq, lhsImag, rhsImag); rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp); return mlir::success(); } - auto realCmp = rewriter.create<mlir::LLVM::FCmpOp>( - loc, mlir::LLVM::FCmpPredicate::oeq, lhsReal, rhsReal); - auto imagCmp = rewriter.create<mlir::LLVM::FCmpOp>( - loc, mlir::LLVM::FCmpPredicate::oeq, lhsImag, rhsImag); + auto realCmp = mlir::LLVM::FCmpOp::create( + rewriter, loc, mlir::LLVM::FCmpPredicate::oeq, lhsReal, rhsReal); + auto imagCmp = mlir::LLVM::FCmpOp::create( + rewriter, loc, mlir::LLVM::FCmpPredicate::oeq, lhsImag, rhsImag); rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp); return mlir::success(); } if (cmpOp.getKind() == cir::CmpOpKind::ne) { if (complexElemTy.isInteger()) { - auto realCmp = rewriter.create<mlir::LLVM::ICmpOp>( - loc, mlir::LLVM::ICmpPredicate::ne, lhsReal, rhsReal); - auto imagCmp = rewriter.create<mlir::LLVM::ICmpOp>( - loc, mlir::LLVM::ICmpPredicate::ne, lhsImag, rhsImag); + auto realCmp = mlir::LLVM::ICmpOp::create( + rewriter, loc, mlir::LLVM::ICmpPredicate::ne, lhsReal, rhsReal); + auto imagCmp = mlir::LLVM::ICmpOp::create( + rewriter, loc, mlir::LLVM::ICmpPredicate::ne, lhsImag, rhsImag); rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp); return mlir::success(); } - auto realCmp = rewriter.create<mlir::LLVM::FCmpOp>( - loc, mlir::LLVM::FCmpPredicate::une, lhsReal, rhsReal); - auto imagCmp = rewriter.create<mlir::LLVM::FCmpOp>( - loc, mlir::LLVM::FCmpPredicate::une, lhsImag, rhsImag); + auto realCmp = mlir::LLVM::FCmpOp::create( + rewriter, loc, mlir::LLVM::FCmpPredicate::une, lhsReal, rhsReal); + auto imagCmp = mlir::LLVM::FCmpOp::create( + rewriter, loc, mlir::LLVM::FCmpPredicate::une, lhsImag, rhsImag); rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp); return mlir::success(); } @@ -2725,7 +2741,7 @@ static void buildCtorDtorList( index); } - builder.create<mlir::LLVM::ReturnOp>(loc, result); + mlir::LLVM::ReturnOp::create(builder, loc, result); } // The applyPartialConversion function traverses blocks in the dominance order, @@ -2904,7 +2920,7 @@ void createLLVMFuncOpIfNotExist(mlir::ConversionPatternRewriter &rewriter, if (!sourceSymbol) { mlir::OpBuilder::InsertionGuard guard(rewriter); rewriter.setInsertionPoint(enclosingFnOp); - rewriter.create<mlir::LLVM::LLVMFuncOp>(srcOp->getLoc(), fnName, fnTy); + mlir::LLVM::LLVMFuncOp::create(rewriter, srcOp->getLoc(), fnName, fnTy); } } @@ -2983,12 +2999,12 @@ mlir::LogicalResult CIRToLLVMTrapOpLowering::matchAndRewrite( mlir::Location loc = op->getLoc(); rewriter.eraseOp(op); - rewriter.create<mlir::LLVM::Trap>(loc); + mlir::LLVM::Trap::create(rewriter, loc); // Note that the call to llvm.trap is not a terminator in LLVM dialect. // So we must emit an additional llvm.unreachable to terminate the current // block. - rewriter.create<mlir::LLVM::UnreachableOp>(loc); + mlir::LLVM::UnreachableOp::create(rewriter, loc); return mlir::success(); } @@ -3114,15 +3130,15 @@ mlir::LogicalResult CIRToLLVMVecCreateOpLowering::matchAndRewrite( const auto vecTy = mlir::cast<cir::VectorType>(op.getType()); const mlir::Type llvmTy = typeConverter->convertType(vecTy); const mlir::Location loc = op.getLoc(); - mlir::Value result = rewriter.create<mlir::LLVM::PoisonOp>(loc, llvmTy); + mlir::Value result = mlir::LLVM::PoisonOp::create(rewriter, loc, llvmTy); assert(vecTy.getSize() == op.getElements().size() && "cir.vec.create op count doesn't match vector type elements count"); for (uint64_t i = 0; i < vecTy.getSize(); ++i) { const mlir::Value indexValue = - rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), i); - result = rewriter.create<mlir::LLVM::InsertElementOp>( - loc, result, adaptor.getElements()[i], indexValue); + mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i); + result = mlir::LLVM::InsertElementOp::create( + rewriter, loc, result, adaptor.getElements()[i], indexValue); } rewriter.replaceOp(op, result); @@ -3151,13 +3167,13 @@ mlir::LogicalResult CIRToLLVMVecCmpOpLowering::matchAndRewrite( mlir::Type elementType = elementTypeIfVector(op.getLhs().getType()); mlir::Value bitResult; if (auto intType = mlir::dyn_cast<cir::IntType>(elementType)) { - bitResult = rewriter.create<mlir::LLVM::ICmpOp>( - op.getLoc(), + bitResult = mlir::LLVM::ICmpOp::create( + rewriter, op.getLoc(), convertCmpKindToICmpPredicate(op.getKind(), intType.isSigned()), adaptor.getLhs(), adaptor.getRhs()); } else if (mlir::isa<cir::FPTypeInterface>(elementType)) { - bitResult = rewriter.create<mlir::LLVM::FCmpOp>( - op.getLoc(), convertCmpKindToFCmpPredicate(op.getKind()), + bitResult = mlir::LLVM::FCmpOp::create( + rewriter, op.getLoc(), convertCmpKindToFCmpPredicate(op.getKind()), adaptor.getLhs(), adaptor.getRhs()); } else { return op.emitError() << "unsupported type for VecCmpOp: " << elementType; @@ -3181,7 +3197,7 @@ mlir::LogicalResult CIRToLLVMVecSplatOpLowering::matchAndRewrite( cir::VectorType vecTy = op.getType(); mlir::Type llvmTy = typeConverter->convertType(vecTy); mlir::Location loc = op.getLoc(); - mlir::Value poison = rewriter.create<mlir::LLVM::PoisonOp>(loc, llvmTy); + mlir::Value poison = mlir::LLVM::PoisonOp::create(rewriter, loc, llvmTy); mlir::Value elementValue = adaptor.getValue(); if (elementValue.getDefiningOp<mlir::LLVM::PoisonOp>()) { @@ -3210,9 +3226,9 @@ mlir::LogicalResult CIRToLLVMVecSplatOpLowering::matchAndRewrite( } mlir::Value indexValue = - rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), 0); - mlir::Value oneElement = rewriter.create<mlir::LLVM::InsertElementOp>( - loc, poison, elementValue, indexValue); + mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), 0); + mlir::Value oneElement = mlir::LLVM::InsertElementOp::create( + rewriter, loc, poison, elementValue, indexValue); SmallVector<int32_t> zeroValues(vecTy.getSize(), 0); rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(op, oneElement, poison, zeroValues); @@ -3260,31 +3276,32 @@ mlir::LogicalResult CIRToLLVMVecShuffleDynamicOpLowering::matchAndRewrite( mlir::cast<cir::VectorType>(op.getVec().getType()).getSize(); uint64_t maskBits = llvm::NextPowerOf2(numElements - 1) - 1; - mlir::Value maskValue = rewriter.create<mlir::LLVM::ConstantOp>( - loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, maskBits)); + mlir::Value maskValue = mlir::LLVM::ConstantOp::create( + rewriter, loc, llvmIndexType, + rewriter.getIntegerAttr(llvmIndexType, maskBits)); mlir::Value maskVector = - rewriter.create<mlir::LLVM::UndefOp>(loc, llvmIndexVecType); + mlir::LLVM::UndefOp::create(rewriter, loc, llvmIndexVecType); for (uint64_t i = 0; i < numElements; ++i) { mlir::Value idxValue = - rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), i); - maskVector = rewriter.create<mlir::LLVM::InsertElementOp>( - loc, maskVector, maskValue, idxValue); + mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i); + maskVector = mlir::LLVM::InsertElementOp::create(rewriter, loc, maskVector, + maskValue, idxValue); } - mlir::Value maskedIndices = rewriter.create<mlir::LLVM::AndOp>( - loc, llvmIndexVecType, adaptor.getIndices(), maskVector); - mlir::Value result = rewriter.create<mlir::LLVM::UndefOp>( - loc, getTypeConverter()->convertType(op.getVec().getType())); + mlir::Value maskedIndices = mlir::LLVM::AndOp::create( + rewriter, loc, llvmIndexVecType, adaptor.getIndices(), maskVector); + mlir::Value result = mlir::LLVM::UndefOp::create( + rewriter, loc, getTypeConverter()->convertType(op.getVec().getType())); for (uint64_t i = 0; i < numElements; ++i) { mlir::Value iValue = - rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), i); - mlir::Value indexValue = rewriter.create<mlir::LLVM::ExtractElementOp>( - loc, maskedIndices, iValue); + mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i); + mlir::Value indexValue = mlir::LLVM::ExtractElementOp::create( + rewriter, loc, maskedIndices, iValue); mlir::Value valueAtIndex = - rewriter.create<mlir::LLVM::ExtractElementOp>(loc, input, indexValue); - result = rewriter.create<mlir::LLVM::InsertElementOp>(loc, result, - valueAtIndex, iValue); + mlir::LLVM::ExtractElementOp::create(rewriter, loc, input, indexValue); + result = mlir::LLVM::InsertElementOp::create(rewriter, loc, result, + valueAtIndex, iValue); } rewriter.replaceOp(op, result); return mlir::success(); @@ -3294,10 +3311,10 @@ mlir::LogicalResult CIRToLLVMVecTernaryOpLowering::matchAndRewrite( cir::VecTernaryOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const { // Convert `cond` into a vector of i1, then use that in a `select` op. - mlir::Value bitVec = rewriter.create<mlir::LLVM::ICmpOp>( - op.getLoc(), mlir::LLVM::ICmpPredicate::ne, adaptor.getCond(), - rewriter.create<mlir::LLVM::ZeroOp>( - op.getCond().getLoc(), + mlir::Value bitVec = mlir::LLVM::ICmpOp::create( + rewriter, op.getLoc(), mlir::LLVM::ICmpPredicate::ne, adaptor.getCond(), + mlir::LLVM::ZeroOp::create( + rewriter, op.getCond().getLoc(), typeConverter->convertType(op.getCond().getType()))); rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>( op, bitVec, adaptor.getLhs(), adaptor.getRhs()); @@ -3314,41 +3331,41 @@ mlir::LogicalResult CIRToLLVMComplexAddOpLowering::matchAndRewrite( auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType()); mlir::Type complexElemTy = getTypeConverter()->convertType(complexType.getElementType()); - auto lhsReal = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0); - auto lhsImag = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1); - auto rhsReal = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0); - auto rhsImag = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1); + auto lhsReal = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0})); + auto lhsImag = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1})); + auto rhsReal = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0})); + auto rhsImag = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1})); mlir::Value newReal; mlir::Value newImag; if (complexElemTy.isInteger()) { - newReal = rewriter.create<mlir::LLVM::AddOp>(loc, complexElemTy, lhsReal, - rhsReal); - newImag = rewriter.create<mlir::LLVM::AddOp>(loc, complexElemTy, lhsImag, - rhsImag); + newReal = mlir::LLVM::AddOp::create(rewriter, loc, complexElemTy, lhsReal, + rhsReal); + newImag = mlir::LLVM::AddOp::create(rewriter, loc, complexElemTy, lhsImag, + rhsImag); } else { assert(!cir::MissingFeatures::fastMathFlags()); assert(!cir::MissingFeatures::fpConstraints()); - newReal = rewriter.create<mlir::LLVM::FAddOp>(loc, complexElemTy, lhsReal, - rhsReal); - newImag = rewriter.create<mlir::LLVM::FAddOp>(loc, complexElemTy, lhsImag, - rhsImag); + newReal = mlir::LLVM::FAddOp::create(rewriter, loc, complexElemTy, lhsReal, + rhsReal); + newImag = mlir::LLVM::FAddOp::create(rewriter, loc, complexElemTy, lhsImag, + rhsImag); } mlir::Type complexLLVMTy = getTypeConverter()->convertType(op.getResult().getType()); auto initialComplex = - rewriter.create<mlir::LLVM::PoisonOp>(op->getLoc(), complexLLVMTy); + mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), complexLLVMTy); - auto realComplex = rewriter.create<mlir::LLVM::InsertValueOp>( - op->getLoc(), initialComplex, newReal, 0); + auto realComplex = mlir::LLVM::InsertValueOp::create( + rewriter, op->getLoc(), initialComplex, newReal, ArrayRef(int64_t{0})); - rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(op, realComplex, - newImag, 1); + rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>( + op, realComplex, newImag, ArrayRef(int64_t{1})); return mlir::success(); } @@ -3359,13 +3376,15 @@ mlir::LogicalResult CIRToLLVMComplexCreateOpLowering::matchAndRewrite( mlir::Type complexLLVMTy = getTypeConverter()->convertType(op.getResult().getType()); auto initialComplex = - rewriter.create<mlir::LLVM::UndefOp>(op->getLoc(), complexLLVMTy); + mlir::LLVM::UndefOp::create(rewriter, op->getLoc(), complexLLVMTy); - auto realComplex = rewriter.create<mlir::LLVM::InsertValueOp>( - op->getLoc(), initialComplex, adaptor.getReal(), 0); + auto realComplex = mlir::LLVM::InsertValueOp::create( + rewriter, op->getLoc(), initialComplex, adaptor.getReal(), + ArrayRef(int64_t{0})); - auto complex = rewriter.create<mlir::LLVM::InsertValueOp>( - op->getLoc(), realComplex, adaptor.getImag(), 1); + auto complex = mlir::LLVM::InsertValueOp::create( + rewriter, op->getLoc(), realComplex, adaptor.getImag(), + ArrayRef(int64_t{1})); rewriter.replaceOp(op, complex); return mlir::success(); @@ -3395,41 +3414,41 @@ mlir::LogicalResult CIRToLLVMComplexSubOpLowering::matchAndRewrite( auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType()); mlir::Type complexElemTy = getTypeConverter()->convertType(complexType.getElementType()); - auto lhsReal = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0); - auto lhsImag = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1); - auto rhsReal = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0); - auto rhsImag = - rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1); + auto lhsReal = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0})); + auto lhsImag = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1})); + auto rhsReal = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0})); + auto rhsImag = mlir::LLVM::ExtractValueOp::create( + rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1})); mlir::Value newReal; mlir::Value newImag; if (complexElemTy.isInteger()) { - newReal = rewriter.create<mlir::LLVM::SubOp>(loc, complexElemTy, lhsReal, - rhsReal); - newImag = rewriter.create<mlir::LLVM::SubOp>(loc, complexElemTy, lhsImag, - rhsImag); + newReal = mlir::LLVM::SubOp::create(rewriter, loc, complexElemTy, lhsReal, + rhsReal); + newImag = mlir::LLVM::SubOp::create(rewriter, loc, complexElemTy, lhsImag, + rhsImag); } else { assert(!cir::MissingFeatures::fastMathFlags()); assert(!cir::MissingFeatures::fpConstraints()); - newReal = rewriter.create<mlir::LLVM::FSubOp>(loc, complexElemTy, lhsReal, - rhsReal); - newImag = rewriter.create<mlir::LLVM::FSubOp>(loc, complexElemTy, lhsImag, - rhsImag); + newReal = mlir::LLVM::FSubOp::create(rewriter, loc, complexElemTy, lhsReal, + rhsReal); + newImag = mlir::LLVM::FSubOp::create(rewriter, loc, complexElemTy, lhsImag, + rhsImag); } mlir::Type complexLLVMTy = getTypeConverter()->convertType(op.getResult().getType()); auto initialComplex = - rewriter.create<mlir::LLVM::PoisonOp>(op->getLoc(), complexLLVMTy); + mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), complexLLVMTy); - auto realComplex = rewriter.create<mlir::LLVM::InsertValueOp>( - op->getLoc(), initialComplex, newReal, 0); + auto realComplex = mlir::LLVM::InsertValueOp::create( + rewriter, op->getLoc(), initialComplex, newReal, ArrayRef(int64_t{0})); - rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(op, realComplex, - newImag, 1); + rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>( + op, realComplex, newImag, ArrayRef(int64_t{1})); return mlir::success(); } @@ -3496,8 +3515,8 @@ mlir::LogicalResult CIRToLLVMSetBitfieldOpLowering::matchAndRewrite( if (storageSize != size) { assert(storageSize > size && "Invalid bitfield size."); - mlir::Value val = rewriter.create<mlir::LLVM::LoadOp>( - op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(), + mlir::Value val = mlir::LLVM::LoadOp::create( + rewriter, op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(), op.getIsVolatile()); srcVal = @@ -3510,11 +3529,11 @@ mlir::LogicalResult CIRToLLVMSetBitfieldOpLowering::matchAndRewrite( ~llvm::APInt::getBitsSet(srcWidth, offset, offset + size)); // Or together the unchanged values and the source value. - srcVal = rewriter.create<mlir::LLVM::OrOp>(op.getLoc(), val, srcVal); + srcVal = mlir::LLVM::OrOp::create(rewriter, op.getLoc(), val, srcVal); } - rewriter.create<mlir::LLVM::StoreOp>(op.getLoc(), srcVal, adaptor.getAddr(), - op.getAlignment(), op.getIsVolatile()); + mlir::LLVM::StoreOp::create(rewriter, op.getLoc(), srcVal, adaptor.getAddr(), + op.getAlignment(), op.getIsVolatile()); mlir::Type resultTy = getTypeConverter()->convertType(op.getType()); @@ -3587,10 +3606,10 @@ mlir::LogicalResult CIRToLLVMGetBitfieldOpLowering::matchAndRewrite( mlir::IntegerType intType = computeBitfieldIntType(storageType, context, storageSize); - mlir::Value val = rewriter.create<mlir::LLVM::LoadOp>( - op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(), + mlir::Value val = mlir::LLVM::LoadOp::create( + rewriter, op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(), op.getIsVolatile()); - val = rewriter.create<mlir::LLVM::BitcastOp>(op.getLoc(), intType, val); + val = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), intType, val); if (info.getIsSigned()) { assert(static_cast<unsigned>(offset + size) <= storageSize); diff --git a/clang/lib/CIR/Lowering/LoweringHelpers.cpp b/clang/lib/CIR/Lowering/LoweringHelpers.cpp index d5f1324..0786579 100644 --- a/clang/lib/CIR/Lowering/LoweringHelpers.cpp +++ b/clang/lib/CIR/Lowering/LoweringHelpers.cpp @@ -148,37 +148,37 @@ lowerConstArrayAttr(cir::ConstArrayAttr constArr, mlir::Value getConstAPInt(mlir::OpBuilder &bld, mlir::Location loc, mlir::Type typ, const llvm::APInt &val) { - return bld.create<mlir::LLVM::ConstantOp>(loc, typ, val); + return mlir::LLVM::ConstantOp::create(bld, loc, typ, val); } mlir::Value getConst(mlir::OpBuilder &bld, mlir::Location loc, mlir::Type typ, unsigned val) { - return bld.create<mlir::LLVM::ConstantOp>(loc, typ, val); + return mlir::LLVM::ConstantOp::create(bld, loc, typ, val); } mlir::Value createShL(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) { if (!rhs) return lhs; mlir::Value rhsVal = getConst(bld, lhs.getLoc(), lhs.getType(), rhs); - return bld.create<mlir::LLVM::ShlOp>(lhs.getLoc(), lhs, rhsVal); + return mlir::LLVM::ShlOp::create(bld, lhs.getLoc(), lhs, rhsVal); } mlir::Value createAShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) { if (!rhs) return lhs; mlir::Value rhsVal = getConst(bld, lhs.getLoc(), lhs.getType(), rhs); - return bld.create<mlir::LLVM::AShrOp>(lhs.getLoc(), lhs, rhsVal); + return mlir::LLVM::AShrOp::create(bld, lhs.getLoc(), lhs, rhsVal); } mlir::Value createAnd(mlir::OpBuilder &bld, mlir::Value lhs, const llvm::APInt &rhs) { mlir::Value rhsVal = getConstAPInt(bld, lhs.getLoc(), lhs.getType(), rhs); - return bld.create<mlir::LLVM::AndOp>(lhs.getLoc(), lhs, rhsVal); + return mlir::LLVM::AndOp::create(bld, lhs.getLoc(), lhs, rhsVal); } mlir::Value createLShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) { if (!rhs) return lhs; mlir::Value rhsVal = getConst(bld, lhs.getLoc(), lhs.getType(), rhs); - return bld.create<mlir::LLVM::LShrOp>(lhs.getLoc(), lhs, rhsVal); + return mlir::LLVM::LShrOp::create(bld, lhs.getLoc(), lhs, rhsVal); } diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 25971d2..c97a9e8 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -3791,18 +3791,12 @@ static bool isFunctionDeclarationName(const LangOptions &LangOpts, if (Current.is(TT_FunctionDeclarationName)) return true; - if (!Current.Tok.getIdentifierInfo()) + if (Current.isNoneOf(tok::identifier, tok::kw_operator)) return false; const auto *Prev = Current.getPreviousNonComment(); assert(Prev); - if (Prev->is(tok::coloncolon)) - Prev = Prev->Previous; - - if (!Prev) - return false; - const auto &Previous = *Prev; if (const auto *PrevPrev = Previous.getPreviousNonComment(); @@ -3851,6 +3845,8 @@ static bool isFunctionDeclarationName(const LangOptions &LangOpts, // Find parentheses of parameter list. if (Current.is(tok::kw_operator)) { + if (Line.startsWith(tok::kw_friend)) + return true; if (Previous.Tok.getIdentifierInfo() && Previous.isNoneOf(tok::kw_return, tok::kw_co_return)) { return true; diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp index d53b64a..6cc7094 100644 --- a/clang/lib/Frontend/ASTUnit.cpp +++ b/clang/lib/Frontend/ASTUnit.cpp @@ -512,152 +512,73 @@ namespace { /// Gathers information from ASTReader that will be used to initialize /// a Preprocessor. class ASTInfoCollector : public ASTReaderListener { - Preprocessor &PP; - ASTContext *Context; HeaderSearchOptions &HSOpts; + std::string &SpecificModuleCachePath; PreprocessorOptions &PPOpts; - LangOptions &LangOpt; + LangOptions &LangOpts; CodeGenOptions &CodeGenOpts; - std::shared_ptr<TargetOptions> &TargetOpts; - IntrusiveRefCntPtr<TargetInfo> &Target; + TargetOptions &TargetOpts; unsigned &Counter; - bool InitializedLanguage = false; - bool InitializedHeaderSearchPaths = false; public: - ASTInfoCollector(Preprocessor &PP, ASTContext *Context, - HeaderSearchOptions &HSOpts, PreprocessorOptions &PPOpts, - LangOptions &LangOpt, CodeGenOptions &CodeGenOpts, - std::shared_ptr<TargetOptions> &TargetOpts, - IntrusiveRefCntPtr<TargetInfo> &Target, unsigned &Counter) - : PP(PP), Context(Context), HSOpts(HSOpts), PPOpts(PPOpts), - LangOpt(LangOpt), CodeGenOpts(CodeGenOpts), TargetOpts(TargetOpts), - Target(Target), Counter(Counter) {} - - bool ReadLanguageOptions(const LangOptions &LangOpts, + ASTInfoCollector(HeaderSearchOptions &HSOpts, + std::string &SpecificModuleCachePath, + PreprocessorOptions &PPOpts, LangOptions &LangOpts, + CodeGenOptions &CodeGenOpts, TargetOptions &TargetOpts, + unsigned &Counter) + : HSOpts(HSOpts), SpecificModuleCachePath(SpecificModuleCachePath), + PPOpts(PPOpts), LangOpts(LangOpts), CodeGenOpts(CodeGenOpts), + TargetOpts(TargetOpts), Counter(Counter) {} + + bool ReadLanguageOptions(const LangOptions &NewLangOpts, StringRef ModuleFilename, bool Complain, bool AllowCompatibleDifferences) override { - if (InitializedLanguage) - return false; - - // FIXME: We did similar things in ReadHeaderSearchOptions too. But such - // style is not scaling. Probably we need to invite some mechanism to - // handle such patterns generally. - auto PICLevel = LangOpt.PICLevel; - auto PIE = LangOpt.PIE; - - LangOpt = LangOpts; - - LangOpt.PICLevel = PICLevel; - LangOpt.PIE = PIE; - - InitializedLanguage = true; - - updated(); + LangOpts = NewLangOpts; return false; } - bool ReadCodeGenOptions(const CodeGenOptions &CGOpts, + bool ReadCodeGenOptions(const CodeGenOptions &NewCodeGenOpts, StringRef ModuleFilename, bool Complain, bool AllowCompatibleDifferences) override { - this->CodeGenOpts = CGOpts; + CodeGenOpts = NewCodeGenOpts; return false; } - bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts, + bool ReadHeaderSearchOptions(const HeaderSearchOptions &NewHSOpts, StringRef ModuleFilename, - StringRef SpecificModuleCachePath, + StringRef NewSpecificModuleCachePath, bool Complain) override { - // llvm::SaveAndRestore doesn't support bit field. - auto ForceCheckCXX20ModulesInputFiles = - this->HSOpts.ForceCheckCXX20ModulesInputFiles; - llvm::SaveAndRestore X(this->HSOpts.UserEntries); - llvm::SaveAndRestore Y(this->HSOpts.SystemHeaderPrefixes); - llvm::SaveAndRestore Z(this->HSOpts.VFSOverlayFiles); - - this->HSOpts = HSOpts; - this->HSOpts.ForceCheckCXX20ModulesInputFiles = - ForceCheckCXX20ModulesInputFiles; - + HSOpts = NewHSOpts; + SpecificModuleCachePath = NewSpecificModuleCachePath; return false; } - bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts, + bool ReadHeaderSearchPaths(const HeaderSearchOptions &NewHSOpts, bool Complain) override { - if (InitializedHeaderSearchPaths) - return false; - - this->HSOpts.UserEntries = HSOpts.UserEntries; - this->HSOpts.SystemHeaderPrefixes = HSOpts.SystemHeaderPrefixes; - this->HSOpts.VFSOverlayFiles = HSOpts.VFSOverlayFiles; - - // Initialize the FileManager. We can't do this in update(), since that - // performs the initialization too late (once both target and language - // options are read). - PP.getFileManager().setVirtualFileSystem(createVFSFromOverlayFiles( - HSOpts.VFSOverlayFiles, PP.getDiagnostics(), - PP.getFileManager().getVirtualFileSystemPtr())); - - InitializedHeaderSearchPaths = true; - + HSOpts.UserEntries = NewHSOpts.UserEntries; + HSOpts.SystemHeaderPrefixes = NewHSOpts.SystemHeaderPrefixes; + HSOpts.VFSOverlayFiles = NewHSOpts.VFSOverlayFiles; return false; } - bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts, + bool ReadPreprocessorOptions(const PreprocessorOptions &NewPPOpts, StringRef ModuleFilename, bool ReadMacros, bool Complain, std::string &SuggestedPredefines) override { - this->PPOpts = PPOpts; + PPOpts = NewPPOpts; return false; } - bool ReadTargetOptions(const TargetOptions &TargetOpts, + bool ReadTargetOptions(const TargetOptions &NewTargetOpts, StringRef ModuleFilename, bool Complain, bool AllowCompatibleDifferences) override { - // If we've already initialized the target, don't do it again. - if (Target) - return false; - - this->TargetOpts = std::make_shared<TargetOptions>(TargetOpts); - Target = - TargetInfo::CreateTargetInfo(PP.getDiagnostics(), *this->TargetOpts); - - updated(); + TargetOpts = NewTargetOpts; return false; } void ReadCounter(const serialization::ModuleFile &M, - unsigned Value) override { - Counter = Value; - } - -private: - void updated() { - if (!Target || !InitializedLanguage) - return; - - // Inform the target of the language options. - // - // FIXME: We shouldn't need to do this, the target should be immutable once - // created. This complexity should be lifted elsewhere. - Target->adjust(PP.getDiagnostics(), LangOpt, /*AuxTarget=*/nullptr); - - // Initialize the preprocessor. - PP.Initialize(*Target); - - if (!Context) - return; - - // Initialize the ASTContext - Context->InitBuiltinTypes(*Target); - - // Adjust printing policy based on language options. - Context->setPrintingPolicy(PrintingPolicy(LangOpt)); - - // We didn't have access to the comment options when the ASTContext was - // constructed, so register them now. - Context->getCommentCommandTraits().registerCommentOptions( - LangOpt.CommentOpts); + unsigned NewCounter) override { + Counter = NewCounter; } }; @@ -812,7 +733,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( std::shared_ptr<DiagnosticOptions> DiagOpts, IntrusiveRefCntPtr<DiagnosticsEngine> Diags, const FileSystemOptions &FileSystemOpts, const HeaderSearchOptions &HSOpts, - const LangOptions *LangOpts, bool OnlyLocalDecls, + const LangOptions *ProvidedLangOpts, bool OnlyLocalDecls, CaptureDiagsKind CaptureDiagnostics, bool AllowASTWithCompilerErrors, bool UserFilesAreVolatile) { std::unique_ptr<ASTUnit> AST(new ASTUnit(true)); @@ -826,41 +747,71 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( ConfigureDiags(Diags, *AST, CaptureDiagnostics); - AST->LangOpts = LangOpts ? std::make_unique<LangOptions>(*LangOpts) - : std::make_unique<LangOptions>(); + std::unique_ptr<LangOptions> LocalLangOpts; + const LangOptions &LangOpts = [&]() -> const LangOptions & { + if (ProvidedLangOpts) + return *ProvidedLangOpts; + LocalLangOpts = std::make_unique<LangOptions>(); + return *LocalLangOpts; + }(); + + AST->LangOpts = std::make_unique<LangOptions>(LangOpts); AST->OnlyLocalDecls = OnlyLocalDecls; AST->CaptureDiagnostics = CaptureDiagnostics; AST->DiagOpts = DiagOpts; AST->Diagnostics = Diags; - AST->FileMgr = llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOpts, VFS); AST->UserFilesAreVolatile = UserFilesAreVolatile; - AST->SourceMgr = llvm::makeIntrusiveRefCnt<SourceManager>( - AST->getDiagnostics(), AST->getFileManager(), UserFilesAreVolatile); - AST->ModCache = createCrossProcessModuleCache(); AST->HSOpts = std::make_unique<HeaderSearchOptions>(HSOpts); AST->HSOpts->ModuleFormat = std::string(PCHContainerRdr.getFormats().front()); - AST->HeaderInfo.reset(new HeaderSearch(AST->getHeaderSearchOpts(), - AST->getSourceManager(), - AST->getDiagnostics(), - AST->getLangOpts(), - /*Target=*/nullptr)); AST->PPOpts = std::make_shared<PreprocessorOptions>(); + AST->CodeGenOpts = std::make_unique<CodeGenOptions>(); + AST->TargetOpts = std::make_shared<TargetOptions>(); + + AST->ModCache = createCrossProcessModuleCache(); + + // Gather info for preprocessor construction later on. + std::string SpecificModuleCachePath; + unsigned Counter = 0; + // Using a temporary FileManager since the AST file might specify custom + // HeaderSearchOptions::VFSOverlayFiles that affect the underlying VFS. + FileManager TmpFileMgr(FileSystemOpts, VFS); + ASTInfoCollector Collector(*AST->HSOpts, SpecificModuleCachePath, + *AST->PPOpts, *AST->LangOpts, *AST->CodeGenOpts, + *AST->TargetOpts, Counter); + if (ASTReader::readASTFileControlBlock( + Filename, TmpFileMgr, *AST->ModCache, PCHContainerRdr, + /*FindModuleFileExtensions=*/true, Collector, + /*ValidateDiagnosticOptions=*/true, ASTReader::ARR_None)) { + AST->getDiagnostics().Report(diag::err_fe_unable_to_load_pch); + return nullptr; + } + + VFS = createVFSFromOverlayFiles(AST->HSOpts->VFSOverlayFiles, + *AST->Diagnostics, std::move(VFS)); - // Gather Info for preprocessor construction later on. + AST->FileMgr = llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOpts, VFS); + + AST->SourceMgr = llvm::makeIntrusiveRefCnt<SourceManager>( + AST->getDiagnostics(), AST->getFileManager(), UserFilesAreVolatile); - HeaderSearch &HeaderInfo = *AST->HeaderInfo; + AST->HSOpts->PrebuiltModuleFiles = HSOpts.PrebuiltModuleFiles; + AST->HSOpts->PrebuiltModulePaths = HSOpts.PrebuiltModulePaths; + AST->HeaderInfo = std::make_unique<HeaderSearch>( + AST->getHeaderSearchOpts(), AST->getSourceManager(), + AST->getDiagnostics(), AST->getLangOpts(), + /*Target=*/nullptr); + AST->HeaderInfo->setModuleCachePath(SpecificModuleCachePath); AST->PP = std::make_shared<Preprocessor>( *AST->PPOpts, AST->getDiagnostics(), *AST->LangOpts, - AST->getSourceManager(), HeaderInfo, AST->ModuleLoader, + AST->getSourceManager(), *AST->HeaderInfo, AST->ModuleLoader, /*IILookup=*/nullptr, /*OwnsHeaderSearch=*/false); - Preprocessor &PP = *AST->PP; if (ToLoad >= LoadASTOnly) AST->Ctx = llvm::makeIntrusiveRefCnt<ASTContext>( - *AST->LangOpts, AST->getSourceManager(), PP.getIdentifierTable(), - PP.getSelectorTable(), PP.getBuiltinInfo(), + *AST->LangOpts, AST->getSourceManager(), AST->PP->getIdentifierTable(), + AST->PP->getSelectorTable(), AST->PP->getBuiltinInfo(), AST->getTranslationUnitKind()); DisableValidationForModuleKind disableValid = @@ -868,24 +819,60 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION")) disableValid = DisableValidationForModuleKind::All; AST->Reader = llvm::makeIntrusiveRefCnt<ASTReader>( - PP, *AST->ModCache, AST->Ctx.get(), PCHContainerRdr, *AST->CodeGenOpts, - ArrayRef<std::shared_ptr<ModuleFileExtension>>(), + *AST->PP, *AST->ModCache, AST->Ctx.get(), PCHContainerRdr, + *AST->CodeGenOpts, ArrayRef<std::shared_ptr<ModuleFileExtension>>(), /*isysroot=*/"", /*DisableValidationKind=*/disableValid, AllowASTWithCompilerErrors); - unsigned Counter = 0; - AST->Reader->setListener(std::make_unique<ASTInfoCollector>( - *AST->PP, AST->Ctx.get(), *AST->HSOpts, *AST->PPOpts, *AST->LangOpts, - *AST->CodeGenOpts, AST->TargetOpts, AST->Target, Counter)); - - // Attach the AST reader to the AST context as an external AST - // source, so that declarations will be deserialized from the - // AST file as needed. + // Attach the AST reader to the AST context as an external AST source, so that + // declarations will be deserialized from the AST file as needed. // We need the external source to be set up before we read the AST, because // eagerly-deserialized declarations may use it. if (AST->Ctx) AST->Ctx->setExternalSource(AST->Reader); + AST->Target = + TargetInfo::CreateTargetInfo(AST->PP->getDiagnostics(), *AST->TargetOpts); + // Inform the target of the language options. + // + // FIXME: We shouldn't need to do this, the target should be immutable once + // created. This complexity should be lifted elsewhere. + AST->Target->adjust(AST->PP->getDiagnostics(), *AST->LangOpts, + /*AuxTarget=*/nullptr); + + // Initialize the preprocessor. + AST->PP->Initialize(*AST->Target); + + AST->PP->setCounterValue(Counter); + + if (AST->Ctx) { + // Initialize the ASTContext + AST->Ctx->InitBuiltinTypes(*AST->Target); + + // Adjust printing policy based on language options. + AST->Ctx->setPrintingPolicy(PrintingPolicy(*AST->LangOpts)); + + // We didn't have access to the comment options when the ASTContext was + // constructed, so register them now. + AST->Ctx->getCommentCommandTraits().registerCommentOptions( + AST->LangOpts->CommentOpts); + } + + // The temporary FileManager we used for ASTReader::readASTFileControlBlock() + // might have already read stdin, and reading it again will fail. Let's + // explicitly forward the buffer. + if (Filename == "-") + if (auto FE = llvm::expectedToOptional(TmpFileMgr.getSTDIN())) + if (auto BufRef = TmpFileMgr.getBufferForFile(*FE)) { + auto Buf = llvm::MemoryBuffer::getMemBufferCopy( + (*BufRef)->getBuffer(), (*BufRef)->getBufferIdentifier()); + AST->Reader->getModuleManager().addInMemoryBuffer("-", std::move(Buf)); + } + + // Reinstate the provided options that are relevant for reading AST files. + AST->HSOpts->ForceCheckCXX20ModulesInputFiles = + HSOpts.ForceCheckCXX20ModulesInputFiles; + switch (AST->Reader->ReadAST(Filename, serialization::MK_MainFile, SourceLocation(), ASTReader::ARR_None)) { case ASTReader::Success: @@ -901,11 +888,18 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( return nullptr; } - AST->OriginalSourceFile = std::string(AST->Reader->getOriginalSourceFile()); + // Now that we have successfully loaded the AST file, we can reinstate some + // options that the clients expect us to preserve (but would trip AST file + // validation, so we couldn't set them earlier). + AST->HSOpts->UserEntries = HSOpts.UserEntries; + AST->HSOpts->SystemHeaderPrefixes = HSOpts.SystemHeaderPrefixes; + AST->HSOpts->VFSOverlayFiles = HSOpts.VFSOverlayFiles; + AST->LangOpts->PICLevel = LangOpts.PICLevel; + AST->LangOpts->PIE = LangOpts.PIE; - PP.setCounterValue(Counter); + AST->OriginalSourceFile = std::string(AST->Reader->getOriginalSourceFile()); - Module *M = HeaderInfo.lookupModule(AST->getLangOpts().CurrentModule); + Module *M = AST->HeaderInfo->lookupModule(AST->getLangOpts().CurrentModule); if (M && AST->getLangOpts().isCompilingModule() && M->isNamedModule()) AST->Ctx->setCurrentNamedModule(M); @@ -915,13 +909,14 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( // Create a semantic analysis object and tell the AST reader about it. if (ToLoad >= LoadEverything) { - AST->TheSema.reset(new Sema(PP, *AST->Ctx, *AST->Consumer)); + AST->TheSema = std::make_unique<Sema>(*AST->PP, *AST->Ctx, *AST->Consumer); AST->TheSema->Initialize(); AST->Reader->InitializeSema(*AST->TheSema); } // Tell the diagnostic client that we have started a source file. - AST->getDiagnostics().getClient()->BeginSourceFile(PP.getLangOpts(), &PP); + AST->getDiagnostics().getClient()->BeginSourceFile(AST->PP->getLangOpts(), + AST->PP.get()); return AST; } diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index ec01faf..a6fc676 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -708,7 +708,7 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result, } // Late template parsing can begin. - Actions.SetLateTemplateParser(LateTemplateParserCallback, nullptr, this); + Actions.SetLateTemplateParser(LateTemplateParserCallback, this); Actions.ActOnEndOfTranslationUnit(); //else don't tell Sema that we ended parsing: more input might come. return true; diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp index 8ed3df7..23bf7f2 100644 --- a/clang/lib/Sema/Sema.cpp +++ b/clang/lib/Sema/Sema.cpp @@ -276,10 +276,9 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer, Context(ctxt), Consumer(consumer), Diags(PP.getDiagnostics()), SourceMgr(PP.getSourceManager()), APINotes(SourceMgr, LangOpts), AnalysisWarnings(*this), ThreadSafetyDeclCache(nullptr), - LateTemplateParser(nullptr), LateTemplateParserCleanup(nullptr), - OpaqueParser(nullptr), CurContext(nullptr), ExternalSource(nullptr), - StackHandler(Diags), CurScope(nullptr), Ident_super(nullptr), - AMDGPUPtr(std::make_unique<SemaAMDGPU>(*this)), + LateTemplateParser(nullptr), OpaqueParser(nullptr), CurContext(nullptr), + ExternalSource(nullptr), StackHandler(Diags), CurScope(nullptr), + Ident_super(nullptr), AMDGPUPtr(std::make_unique<SemaAMDGPU>(*this)), ARMPtr(std::make_unique<SemaARM>(*this)), AVRPtr(std::make_unique<SemaAVR>(*this)), BPFPtr(std::make_unique<SemaBPF>(*this)), @@ -1248,9 +1247,6 @@ void Sema::ActOnEndOfTranslationUnit() { ? TUFragmentKind::Private : TUFragmentKind::Normal); - if (LateTemplateParserCleanup) - LateTemplateParserCleanup(OpaqueParser); - CheckDelayedMemberExceptionSpecs(); } else { // If we are building a TU prefix for serialization, it is safe to transfer diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 8b3fd41..c1b5cb7 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -5811,7 +5811,13 @@ bool ASTReader::readASTFileControlBlock( // FIXME: This allows use of the VFS; we do not allow use of the // VFS when actually loading a module. - auto BufferOrErr = FileMgr.getBufferForFile(Filename); + auto Entry = + Filename == "-" ? FileMgr.getSTDIN() : FileMgr.getFileRef(Filename); + if (!Entry) { + llvm::consumeError(Entry.takeError()); + return true; + } + auto BufferOrErr = FileMgr.getBufferForFile(*Entry); if (!BufferOrErr) return true; OwnedBuffer = std::move(*BufferOrErr); diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp index b0096d8..05d5669 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp @@ -382,7 +382,8 @@ DignosticsEngineWithDiagOpts::DignosticsEngineWithDiagOpts( std::pair<std::unique_ptr<driver::Driver>, std::unique_ptr<driver::Compilation>> buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, - IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) { + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, + llvm::BumpPtrAllocator &Alloc) { SmallVector<const char *, 256> Argv; Argv.reserve(ArgStrs.size()); for (const std::string &Arg : ArgStrs) @@ -393,7 +394,6 @@ buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, "clang LLVM compiler", FS); Driver->setTitle("clang_based_tool"); - llvm::BumpPtrAllocator Alloc; bool CLMode = driver::IsClangCL( driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1))); diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h index 71c6731..5657317 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h +++ b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h @@ -105,7 +105,8 @@ struct TextDiagnosticsPrinterWithOutput { std::pair<std::unique_ptr<driver::Driver>, std::unique_ptr<driver::Compilation>> buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, - IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS); + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, + llvm::BumpPtrAllocator &Alloc); std::unique_ptr<CompilerInvocation> createCompilerInvocation(ArrayRef<std::string> CommandLine, diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 9515421..0a1cf6b 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -78,8 +78,10 @@ static bool forEachDriverJob( IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, llvm::function_ref<bool(const driver::Command &Cmd)> Callback) { // Compilation holds a non-owning a reference to the Driver, hence we need to - // keep the Driver alive when we use Compilation. - auto [Driver, Compilation] = buildCompilation(ArgStrs, Diags, FS); + // keep the Driver alive when we use Compilation. Arguments to commands may be + // owned by Alloc when expanded from response files. + llvm::BumpPtrAllocator Alloc; + auto [Driver, Compilation] = buildCompilation(ArgStrs, Diags, FS, Alloc); if (!Compilation) return false; for (const driver::Command &Job : Compilation->getJobs()) { diff --git a/clang/test/CIR/CodeGen/call-via-class-member-funcptr.cpp b/clang/test/CIR/CodeGen/call-via-class-member-funcptr.cpp new file mode 100644 index 0000000..dbde454 --- /dev/null +++ b/clang/test/CIR/CodeGen/call-via-class-member-funcptr.cpp @@ -0,0 +1,86 @@ +// RUN: %clang_cc1 -std=c++20 -triple aarch64-none-linux-android21 -fclangir -emit-cir %s -o %t.cir +// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR +// RUN: %clang_cc1 -std=c++20 -triple aarch64-none-linux-android21 -fclangir -emit-llvm %s -o %t-cir.ll +// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=LLVM +// RUN: %clang_cc1 -std=c++20 -triple aarch64-none-linux-android21 -emit-llvm %s -o %t.ll +// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG + +class A { +public: + static char *b(int); +}; + +int h=0; + +class F { +public: + const char *b(); + A g; +}; + +const char *F::b() { return g.b(h); } + +void fn1() { F f1; } + +// CIR: cir.func {{.*}} @_ZN1F1bEv +// CIR: %[[H_PTR:.*]] = cir.get_global @h : !cir.ptr<!s32i> +// CIR: %[[H_VAL:.*]] = cir.load{{.*}} %[[H_PTR]] : !cir.ptr<!s32i>, !s32i +// CIR: %[[RET:.*]] = cir.call @_ZN1A1bEi(%[[H_VAL]]) : (!s32i) -> !cir.ptr<!s8i> + +// LLVM: define {{.*}} ptr @_ZN1F1bEv +// LLVM: %[[VAR_H:.*]] = load i32, ptr @h +// LLVM: %[[RET:.*]] = call ptr @_ZN1A1bEi(i32 %[[VAR_H]]) + +// OGCG: define {{.*}} ptr @_ZN1F1bEv +// OGCG: %[[VAR_H:.*]] = load i32, ptr @h +// OGCG: %[[RET:.*]] = call noundef ptr @_ZN1A1bEi(i32 noundef %[[VAR_H]]) + +class B { +public: + B(); + int (&indirect_callee_int_ref)(int); +}; + +class C { +public: + int call_indirect(int v) { return inner.indirect_callee_int_ref(v); }; + B inner; +}; + +void fn2() { C c1; c1.call_indirect(2); } + +// CIR: cir.func {{.*}} @_ZN1C13call_indirectEi(%[[THIS_ARG:.*]]: !cir.ptr<!rec_C> {{.*}}, %[[V_ARG:.*]]: !s32i {{.*}}) -> !s32i +// CIR: %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["this", init] +// CIR: %[[V_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["v", init] +// CIR: cir.store %[[THIS_ARG]], %[[THIS_ADDR]] +// CIR: cir.store %[[V_ARG]], %[[V_ADDR]] +// CIR: %[[THIS:.*]] = cir.load %[[THIS_ADDR]] +// CIR: %[[INNER:.*]] = cir.get_member %[[THIS]][0] {name = "inner"} : !cir.ptr<!rec_C> -> !cir.ptr<!rec_B> +// CIR: %[[INDIRECT_CALLEE_PTR:.*]] = cir.get_member %[[INNER]][0] {name = "indirect_callee_int_ref"} +// CIR: %[[INDIRECT_CALLEE:.*]] = cir.load %[[INDIRECT_CALLEE_PTR]] +// CIR: %[[V:.*]] = cir.load{{.*}} %[[V_ADDR]] : !cir.ptr<!s32i>, !s32i +// CIR: %[[RET:.*]] = cir.call %[[INDIRECT_CALLEE]](%[[V]]) + +// LLVM: define {{.*}} i32 @_ZN1C13call_indirectEi(ptr %[[THIS_ARG:.*]], i32 %[[V_ARG:.*]]) +// LLVM: %[[THIS_ADDR:.*]] = alloca ptr +// LLVM: %[[V_ADDR:.*]] = alloca i32 +// LLVM: store ptr %[[THIS_ARG]], ptr %[[THIS_ADDR]] +// LLVM: store i32 %[[V_ARG]], ptr %[[V_ADDR]] +// LLVM: %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]] +// LLVM: %[[INNER:.*]] = getelementptr %class.C, ptr %[[THIS]], i32 0, i32 0 +// LLVM: %[[INDIRECT_CALLEE_PTR:.*]] = getelementptr %class.B, ptr %[[INNER]], i32 0, i32 0 +// LLVM: %[[INDIRECT_CALLEE:.*]] = load ptr, ptr %[[INDIRECT_CALLEE_PTR]] +// LLVM: %[[V:.*]] = load i32, ptr %[[V_ADDR]] +// LLVM: %[[RET:.*]] = call i32 %[[INDIRECT_CALLEE]](i32 %[[V]]) + +// OGCG: define {{.*}} i32 @_ZN1C13call_indirectEi(ptr {{.*}} %[[THIS_ARG:.*]], i32 {{.*}} %[[V_ARG:.*]]) +// OGCG: %[[THIS_ADDR:.*]] = alloca ptr +// OGCG: %[[V_ADDR:.*]] = alloca i32 +// OGCG: store ptr %[[THIS_ARG]], ptr %[[THIS_ADDR]] +// OGCG: store i32 %[[V_ARG]], ptr %[[V_ADDR]] +// OGCG: %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]] +// OGCG: %[[INNER:.*]] = getelementptr inbounds nuw %class.C, ptr %[[THIS]], i32 0, i32 0 +// OGCG: %[[INDIRECT_CALLEE_PTR:.*]] = getelementptr inbounds nuw %class.B, ptr %[[INNER]], i32 0, i32 0 +// OGCG: %[[INDIRECT_CALLEE:.*]] = load ptr, ptr %[[INDIRECT_CALLEE_PTR]] +// OGCG: %[[V:.*]] = load i32, ptr %[[V_ADDR]] +// OGCG: %[[RET:.*]] = call noundef i32 %[[INDIRECT_CALLEE]](i32 noundef %[[V]]) diff --git a/clang/test/CIR/CodeGen/dynamic-cast-exact.cpp b/clang/test/CIR/CodeGen/dynamic-cast-exact.cpp new file mode 100644 index 0000000..e3b8533 --- /dev/null +++ b/clang/test/CIR/CodeGen/dynamic-cast-exact.cpp @@ -0,0 +1,174 @@ +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 -O1 -fclangir -clangir-disable-passes -emit-cir -o %t.cir %s +// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 -O1 -fclangir -emit-llvm -o %t-cir.ll %s +// RUN: FileCheck --input-file=%t-cir.ll --check-prefix=LLVM %s +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 -O1 -emit-llvm -o %t.ll %s +// RUN: FileCheck --input-file=%t.ll --check-prefix=OGCG %s + +struct Base1 { + virtual ~Base1(); +}; + +struct Base2 { + virtual ~Base2(); +}; + +struct Derived final : Base1 {}; + +Derived *ptr_cast(Base1 *ptr) { + return dynamic_cast<Derived *>(ptr); +} + +// CIR: cir.func {{.*}} @_Z8ptr_castP5Base1 +// CIR: %[[SRC:.*]] = cir.load{{.*}} %{{.+}} : !cir.ptr<!cir.ptr<!rec_Base1>>, !cir.ptr<!rec_Base1> +// CIR-NEXT: %[[NULL_PTR:.*]] = cir.const #cir.ptr<null> +// CIR-NEXT: %[[SRC_IS_NULL:.*]] = cir.cmp(eq, %[[SRC]], %[[NULL_PTR]]) +// CIR-NEXT: %[[RESULT:.*]] = cir.ternary(%[[SRC_IS_NULL]], true { +// CIR-NEXT: %[[NULL_PTR_DEST:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!rec_Derived> +// CIR-NEXT: cir.yield %[[NULL_PTR_DEST]] : !cir.ptr<!rec_Derived> +// CIR-NEXT: }, false { +// CIR-NEXT: %[[EXPECTED_VPTR:.*]] = cir.vtable.address_point(@_ZTV7Derived, address_point = <index = 0, offset = 2>) : !cir.vptr +// CIR-NEXT: %[[SRC_VPTR_PTR:.*]] = cir.cast bitcast %[[SRC]] : !cir.ptr<!rec_Base1> -> !cir.ptr<!cir.vptr> +// CIR-NEXT: %[[SRC_VPTR:.*]] = cir.load{{.*}} %[[SRC_VPTR_PTR]] : !cir.ptr<!cir.vptr>, !cir.vptr +// CIR-NEXT: %[[SUCCESS:.*]] = cir.cmp(eq, %[[SRC_VPTR]], %[[EXPECTED_VPTR]]) : !cir.vptr, !cir.bool +// CIR-NEXT: %[[EXACT_RESULT:.*]] = cir.ternary(%[[SUCCESS]], true { +// CIR-NEXT: %[[RES:.*]] = cir.cast bitcast %[[SRC]] : !cir.ptr<!rec_Base1> -> !cir.ptr<!rec_Derived> +// CIR-NEXT: cir.yield %[[RES]] : !cir.ptr<!rec_Derived> +// CIR-NEXT: }, false { +// CIR-NEXT: %[[NULL:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!rec_Derived> +// CIR-NEXT: cir.yield %[[NULL]] : !cir.ptr<!rec_Derived> +// CIR-NEXT: }) : (!cir.bool) -> !cir.ptr<!rec_Derived> +// CIR-NEXT: cir.yield %[[EXACT_RESULT]] : !cir.ptr<!rec_Derived> +// CIR-NEXT: }) : (!cir.bool) -> !cir.ptr<!rec_Derived> + +// Note: The LLVM output omits the label for the entry block (which is +// implicitly %1), so we use %{{.*}} to match the implicit label in the +// phi check. + +// LLVM: define dso_local ptr @_Z8ptr_castP5Base1(ptr{{.*}} %[[SRC:.*]]) +// LLVM-NEXT: %[[SRC_IS_NULL:.*]] = icmp eq ptr %0, null +// LLVM-NEXT: br i1 %[[SRC_IS_NULL]], label %[[LABEL_END:.*]], label %[[LABEL_NOTNULL:.*]] +// LLVM: [[LABEL_NOTNULL]]: +// LLVM-NEXT: %[[VPTR:.*]] = load ptr, ptr %[[SRC]], align 8 +// LLVM-NEXT: %[[SUCCESS:.*]] = icmp eq ptr %[[VPTR]], getelementptr inbounds nuw (i8, ptr @_ZTV7Derived, i64 16) +// LLVM-NEXT: %[[EXACT_RESULT:.*]] = select i1 %[[SUCCESS]], ptr %[[SRC]], ptr null +// LLVM-NEXT: br label %[[LABEL_END]] +// LLVM: [[LABEL_END]]: +// LLVM-NEXT: %[[RESULT:.*]] = phi ptr [ %[[EXACT_RESULT]], %[[LABEL_NOTNULL]] ], [ null, %{{.*}} ] +// LLVM-NEXT: ret ptr %[[RESULT]] +// LLVM-NEXT: } + +// OGCG: define{{.*}} ptr @_Z8ptr_castP5Base1(ptr {{.*}} %[[SRC:.*]]) +// OGCG-NEXT: entry: +// OGCG-NEXT: %[[NULL_CHECK:.*]] = icmp eq ptr %[[SRC]], null +// OGCG-NEXT: br i1 %[[NULL_CHECK]], label %[[LABEL_NULL:.*]], label %[[LABEL_NOTNULL:.*]] +// OGCG: [[LABEL_NOTNULL]]: +// OGCG-NEXT: %[[VTABLE:.*]] = load ptr, ptr %[[SRC]], align 8 +// OGCG-NEXT: %[[VTABLE_CHECK:.*]] = icmp eq ptr %[[VTABLE]], getelementptr inbounds {{.*}} (i8, ptr @_ZTV7Derived, i64 16) +// OGCG-NEXT: br i1 %[[VTABLE_CHECK]], label %[[LABEL_END:.*]], label %[[LABEL_NULL]] +// OGCG: [[LABEL_NULL]]: +// OGCG-NEXT: br label %[[LABEL_END]] +// OGCG: [[LABEL_END]]: +// OGCG-NEXT: %[[RESULT:.*]] = phi ptr [ %[[SRC]], %[[LABEL_NOTNULL]] ], [ null, %[[LABEL_NULL]] ] +// OGCG-NEXT: ret ptr %[[RESULT]] +// OGCG-NEXT: } + +Derived &ref_cast(Base1 &ref) { + return dynamic_cast<Derived &>(ref); +} + +// CIR: cir.func {{.*}} @_Z8ref_castR5Base1 +// CIR: %[[SRC:.*]] = cir.load{{.*}} %{{.+}} : !cir.ptr<!cir.ptr<!rec_Base1>>, !cir.ptr<!rec_Base1> +// CIR-NEXT: %[[EXPECTED_VPTR:.*]] = cir.vtable.address_point(@_ZTV7Derived, address_point = <index = 0, offset = 2>) : !cir.vptr +// CIR-NEXT: %[[SRC_VPTR_PTR:.*]] = cir.cast bitcast %[[SRC]] : !cir.ptr<!rec_Base1> -> !cir.ptr<!cir.vptr> +// CIR-NEXT: %[[SRC_VPTR:.*]] = cir.load{{.*}} %[[SRC_VPTR_PTR]] : !cir.ptr<!cir.vptr>, !cir.vptr +// CIR-NEXT: %[[SUCCESS:.*]] = cir.cmp(eq, %[[SRC_VPTR]], %[[EXPECTED_VPTR]]) : !cir.vptr, !cir.bool +// CIR-NEXT: %[[FAILED:.*]] = cir.unary(not, %[[SUCCESS]]) : !cir.bool, !cir.bool +// CIR-NEXT: cir.if %[[FAILED]] { +// CIR-NEXT: cir.call @__cxa_bad_cast() : () -> () +// CIR-NEXT: cir.unreachable +// CIR-NEXT: } +// CIR-NEXT: %{{.+}} = cir.cast bitcast %[[SRC]] : !cir.ptr<!rec_Base1> -> !cir.ptr<!rec_Derived> + +// LLVM: define{{.*}} ptr @_Z8ref_castR5Base1(ptr{{.*}} %[[SRC:.*]]) +// LLVM-NEXT: %[[VPTR:.*]] = load ptr, ptr %[[SRC]], align 8 +// LLVM-NEXT: %[[OK:.*]] = icmp eq ptr %[[VPTR]], getelementptr inbounds nuw (i8, ptr @_ZTV7Derived, i64 16) +// LLVM-NEXT: br i1 %[[OK]], label %[[LABEL_OK:.*]], label %[[LABEL_FAIL:.*]] +// LLVM: [[LABEL_FAIL]]: +// LLVM-NEXT: tail call void @__cxa_bad_cast() +// LLVM-NEXT: unreachable +// LLVM: [[LABEL_OK]]: +// LLVM-NEXT: ret ptr %[[SRC]] +// LLVM-NEXT: } + +// OGCG: define{{.*}} ptr @_Z8ref_castR5Base1(ptr {{.*}} %[[REF:.*]]) +// OGCG-NEXT: entry: +// OGCG-NEXT: %[[VTABLE:.*]] = load ptr, ptr %[[REF]], align 8 +// OGCG-NEXT: %[[VTABLE_CHECK:.*]] = icmp eq ptr %[[VTABLE]], getelementptr inbounds {{.*}} (i8, ptr @_ZTV7Derived, i64 16) +// OGCG-NEXT: br i1 %[[VTABLE_CHECK]], label %[[LABEL_END:.*]], label %[[LABEL_NULL:.*]] +// OGCG: [[LABEL_NULL]]: +// OGCG-NEXT: {{.*}}call void @__cxa_bad_cast() +// OGCG-NEXT: unreachable +// OGCG: [[LABEL_END]]: +// OGCG-NEXT: ret ptr %[[REF]] +// OGCG-NEXT: } + +struct Offset { virtual ~Offset(); }; +struct A { virtual ~A(); }; +struct B final : Offset, A { }; + +B *offset_cast(A *a) { + return dynamic_cast<B*>(a); +} + +// CIR: cir.func {{.*}} @_Z11offset_castP1A +// CIR: %[[SRC:.*]] = cir.load{{.*}} %{{.+}} : !cir.ptr<!cir.ptr<!rec_A>>, !cir.ptr<!rec_A> +// CIR-NEXT: %[[NULL_PTR:.*]] = cir.const #cir.ptr<null> +// CIR-NEXT: %[[SRC_IS_NULL:.*]] = cir.cmp(eq, %[[SRC]], %[[NULL_PTR]]) +// CIR-NEXT: %[[RESULT:.*]] = cir.ternary(%[[SRC_IS_NULL]], true { +// CIR-NEXT: %[[NULL_PTR_DEST:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!rec_B> +// CIR-NEXT: cir.yield %[[NULL_PTR_DEST]] : !cir.ptr<!rec_B> +// CIR-NEXT: }, false { +// CIR-NEXT: %[[EXPECTED_VPTR:.*]] = cir.vtable.address_point(@_ZTV1B, address_point = <index = 1, offset = 2>) : !cir.vptr +// CIR-NEXT: %[[SRC_VPTR_PTR:.*]] = cir.cast bitcast %[[SRC]] : !cir.ptr<!rec_A> -> !cir.ptr<!cir.vptr> +// CIR-NEXT: %[[SRC_VPTR:.*]] = cir.load{{.*}} %[[SRC_VPTR_PTR]] : !cir.ptr<!cir.vptr>, !cir.vptr +// CIR-NEXT: %[[SUCCESS:.*]] = cir.cmp(eq, %[[SRC_VPTR]], %[[EXPECTED_VPTR]]) : !cir.vptr, !cir.bool +// CIR-NEXT: %[[EXACT_RESULT:.*]] = cir.ternary(%[[SUCCESS]], true { +// CIR-NEXT: %[[MINUS_EIGHT:.*]] = cir.const #cir.int<18446744073709551608> : !u64i +// CIR-NEXT: %[[SRC_VOID:.*]] = cir.cast bitcast %[[SRC]] : !cir.ptr<!rec_A> -> !cir.ptr<!u8i> +// CIR-NEXT: %[[SRC_OFFSET:.*]] = cir.ptr_stride %[[SRC_VOID]], %[[MINUS_EIGHT]] +// CIR-NEXT: %[[RES:.*]] = cir.cast bitcast %[[SRC_OFFSET]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_B> +// CIR-NEXT: cir.yield %[[RES]] : !cir.ptr<!rec_B> +// CIR-NEXT: }, false { +// CIR-NEXT: %[[NULL:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!rec_B> +// CIR-NEXT: cir.yield %[[NULL]] : !cir.ptr<!rec_B> +// CIR-NEXT: }) : (!cir.bool) -> !cir.ptr<!rec_B> +// CIR-NEXT: cir.yield %[[EXACT_RESULT]] : !cir.ptr<!rec_B> +// CIR-NEXT: }) : (!cir.bool) -> !cir.ptr<!rec_B> + +// LLVM: define dso_local ptr @_Z11offset_castP1A(ptr{{.*}} %[[SRC:.*]]) +// LLVM-NEXT: %[[SRC_IS_NULL:.*]] = icmp eq ptr %0, null +// LLVM-NEXT: br i1 %[[SRC_IS_NULL]], label %[[LABEL_END:.*]], label %[[LABEL_NOTNULL:.*]] +// LLVM: [[LABEL_NOTNULL]]: +// LLVM-NEXT: %[[VTABLE:.*]] = load ptr, ptr %[[SRC]] +// LLVM-NEXT: %[[VTABLE_CHECK:.*]] = icmp eq ptr %[[VTABLE]], getelementptr inbounds nuw (i8, ptr @_ZTV1B, i64 48) +// LLVM-NEXT: %[[SRC_OFFSET:.*]] = getelementptr i8, ptr %[[SRC]], i64 -8 +// LLVM-NEXT: %[[EXACT_RESULT:.*]] = select i1 %[[VTABLE_CHECK]], ptr %[[SRC_OFFSET]], ptr null +// LLVM-NEXT: br label %[[LABEL_END]] +// LLVM: [[LABEL_END]]: +// LLVM-NEXT: %[[RESULT:.*]] = phi ptr [ %[[EXACT_RESULT]], %[[LABEL_NOTNULL]] ], [ null, %{{.*}} ] +// LLVM-NEXT: ret ptr %[[RESULT]] +// LLVM-NEXT: } + +// OGCG: define{{.*}} ptr @_Z11offset_castP1A(ptr{{.*}} %[[SRC:.*]]) +// OGCG: %[[SRV_NULL:.*]] = icmp eq ptr %[[SRC]], null +// OGCG-NEXT: br i1 %[[SRV_NULL]], label %[[LABEL_NULL:.*]], label %[[LABEL_NOTNULL:.*]] +// OGCG: [[LABEL_NOTNULL]]: +// OGCG-NEXT: %[[VTABLE:.*]] = load ptr, ptr %[[SRC]] +// OGCG-NEXT: %[[VTABLE_CHECK:.*]] = icmp eq ptr %[[VTABLE]], getelementptr inbounds nuw inrange(-16, 16) (i8, ptr @_ZTV1B, i64 48) +// OGCG-NEXT: %[[RESULT:.*]] = getelementptr inbounds i8, ptr %[[SRC]], i64 -8 +// OGCG-NEXT: br i1 %[[VTABLE_CHECK]], label %[[LABEL_END:.*]], label %[[LABEL_NULL]] +// OGCG: [[LABEL_NULL]]: +// OGCG-NEXT: br label %[[LABEL_END]] +// OGCG: [[LABEL_END]]: +// OGCG-NEXT: phi ptr [ %[[RESULT]], %[[LABEL_NOTNULL]] ], [ null, %[[LABEL_NULL]] ] diff --git a/clang/test/CIR/CodeGen/struct-init.cpp b/clang/test/CIR/CodeGen/struct-init.cpp index cb50999..63e13dd 100644 --- a/clang/test/CIR/CodeGen/struct-init.cpp +++ b/clang/test/CIR/CodeGen/struct-init.cpp @@ -5,6 +5,21 @@ // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll // RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s +struct BitfieldStruct { + unsigned int a:4; + unsigned int b:14; + unsigned int c:14; +}; + +BitfieldStruct overlapping_init = { 3, 2, 1 }; + +// This is unintuitive. The bitfields are initialized using a struct of constants +// that maps to the bitfields but splits the value into bytes. + +// CIR: cir.global external @overlapping_init = #cir.const_record<{#cir.int<35> : !u8i, #cir.int<0> : !u8i, #cir.int<4> : !u8i, #cir.int<0> : !u8i}> : !rec_anon_struct +// LLVM: @overlapping_init = global { i8, i8, i8, i8 } { i8 35, i8 0, i8 4, i8 0 } +// OGCG: @overlapping_init = global { i8, i8, i8, i8 } { i8 35, i8 0, i8 4, i8 0 } + struct S { int a, b, c; }; diff --git a/clang/test/ClangScanDeps/response-file.c b/clang/test/ClangScanDeps/response-file.c index c08105c..f905438 100644 --- a/clang/test/ClangScanDeps/response-file.c +++ b/clang/test/ClangScanDeps/response-file.c @@ -1,10 +1,12 @@ -// Check that the scanner can handle a response file input. +// Check that the scanner can handle a response file input. Uses -verbatim-args +// to ensure response files are expanded by the scanner library and not the +// argumeent adjuster in clang-scan-deps. // RUN: rm -rf %t // RUN: split-file %s %t // RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: clang-scan-deps -format experimental-full -compilation-database %t/cdb.json > %t/deps.json +// RUN: clang-scan-deps -verbatim-args -format experimental-full -compilation-database %t/cdb.json > %t/deps.json // RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s diff --git a/clang/test/CodeGen/allow-ubsan-check-divergence.c b/clang/test/CodeGen/allow-ubsan-check-divergence.c new file mode 100644 index 0000000..a21d4f6 --- /dev/null +++ b/clang/test/CodeGen/allow-ubsan-check-divergence.c @@ -0,0 +1,30 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6 + +// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-llvm -O1 -o - %s \ +// RUN: | FileCheck %s --check-prefixes=CLEAN-O1 + +// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-llvm -O1 -o - %s \ +// RUN: -fsanitize=signed-integer-overflow \ +// RUN: -fsanitize-skip-hot-cutoff=signed-integer-overflow=1.0 \ +// RUN: -fallow-runtime-check-skip-hot-cutoff=1.0 \ +// RUN: | FileCheck %s --check-prefixes=UBSAN-O1 + +// This test shows that -fsanitize-skip-hot-cutoff=...=1.0 plus +// -fallow-runtime-check-skip-hot-cutoff=1.0 does not perfectly undo the +// effects of -fsanitize. + +// CLEAN-O1-LABEL: define dso_local i32 @overflow( +// CLEAN-O1-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { +// CLEAN-O1-NEXT: [[ENTRY:.*:]] +// CLEAN-O1-NEXT: [[ADD:%.*]] = add nsw i32 [[Y]], [[X]] +// CLEAN-O1-NEXT: ret i32 [[ADD]] +// +// UBSAN-O1-LABEL: define dso_local noundef i32 @overflow( +// UBSAN-O1-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { +// UBSAN-O1-NEXT: [[ENTRY:.*:]] +// UBSAN-O1-NEXT: [[TMP0:%.*]] = add i32 [[X]], [[Y]] +// UBSAN-O1-NEXT: ret i32 [[TMP0]] +// +int overflow(int x, int y) { + return x + y; +} diff --git a/clang/test/CodeGenOpenCL/amdgpu-cluster-dims.cl b/clang/test/CodeGenOpenCL/amdgpu-cluster-dims.cl index 8c3e5b7..14fbeb2 100644 --- a/clang/test/CodeGenOpenCL/amdgpu-cluster-dims.cl +++ b/clang/test/CodeGenOpenCL/amdgpu-cluster-dims.cl @@ -26,8 +26,8 @@ kernel void foo(global int *p) { *p = 1; } // CHECK-NEXT: ret void // //. -// CHECK: attributes #[[ATTR0]] = { convergent norecurse nounwind "amdgpu-cluster-dims"="0,0,0" "amdgpu-flat-work-group-size"="1,256" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx1250" "target-features"="+16-bit-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-fmin-fmax-global-f32,+atomic-fmin-fmax-global-f64,+atomic-global-pk-add-bf16-inst,+bf16-cvt-insts,+bf16-pk-insts,+bf16-trans-insts,+bitop3-insts,+ci-insts,+clusters,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+fp8e5m3-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+prng-inst,+setprio-inc-wg-inst,+tanh-insts,+tensor-cvt-lut-insts,+transpose-load-f4f6-insts,+vmem-pref-insts,+wavefrontsize32" "uniform-work-group-size"="false" } -// CHECK: attributes #[[ATTR1]] = { alwaysinline convergent norecurse nounwind "amdgpu-cluster-dims"="0,0,0" "amdgpu-flat-work-group-size"="1,256" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx1250" "target-features"="+16-bit-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-fmin-fmax-global-f32,+atomic-fmin-fmax-global-f64,+atomic-global-pk-add-bf16-inst,+bf16-cvt-insts,+bf16-pk-insts,+bf16-trans-insts,+bitop3-insts,+ci-insts,+clusters,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+fp8e5m3-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+prng-inst,+setprio-inc-wg-inst,+tanh-insts,+tensor-cvt-lut-insts,+transpose-load-f4f6-insts,+vmem-pref-insts,+wavefrontsize32" } +// CHECK: attributes #[[ATTR0]] = { convergent norecurse nounwind "amdgpu-cluster-dims"="0,0,0" "amdgpu-flat-work-group-size"="1,256" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx1250" "target-features"="+16-bit-insts,+add-min-max-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-fmin-fmax-global-f32,+atomic-fmin-fmax-global-f64,+atomic-global-pk-add-bf16-inst,+bf16-cvt-insts,+bf16-pk-insts,+bf16-trans-insts,+bitop3-insts,+ci-insts,+clusters,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+fp8e5m3-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+pk-add-min-max-insts,+prng-inst,+setprio-inc-wg-inst,+tanh-insts,+tensor-cvt-lut-insts,+transpose-load-f4f6-insts,+vmem-pref-insts,+wavefrontsize32" "uniform-work-group-size"="false" } +// CHECK: attributes #[[ATTR1]] = { alwaysinline convergent norecurse nounwind "amdgpu-cluster-dims"="0,0,0" "amdgpu-flat-work-group-size"="1,256" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx1250" "target-features"="+16-bit-insts,+add-min-max-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-fmin-fmax-global-f32,+atomic-fmin-fmax-global-f64,+atomic-global-pk-add-bf16-inst,+bf16-cvt-insts,+bf16-pk-insts,+bf16-trans-insts,+bitop3-insts,+ci-insts,+clusters,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+fp8e5m3-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+pk-add-min-max-insts,+prng-inst,+setprio-inc-wg-inst,+tanh-insts,+tensor-cvt-lut-insts,+transpose-load-f4f6-insts,+vmem-pref-insts,+wavefrontsize32" } // CHECK: attributes #[[ATTR2]] = { convergent nounwind } //. // CHECK: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 600} diff --git a/clang/test/CodeGenOpenCL/amdgpu-features.cl b/clang/test/CodeGenOpenCL/amdgpu-features.cl index 7cc83c0..9bd096f 100644 --- a/clang/test/CodeGenOpenCL/amdgpu-features.cl +++ b/clang/test/CodeGenOpenCL/amdgpu-features.cl @@ -109,8 +109,8 @@ // GFX1153: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+atomic-fmin-fmax-global-f32,+ci-insts,+dl-insts,+dot10-insts,+dot12-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" // GFX1200: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-fmin-fmax-global-f32,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot12-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" // GFX1201: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-fmin-fmax-global-f32,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot12-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" -// GFX1250: "target-features"="+16-bit-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-fmin-fmax-global-f32,+atomic-fmin-fmax-global-f64,+atomic-global-pk-add-bf16-inst,+bf16-cvt-insts,+bf16-pk-insts,+bf16-trans-insts,+bitop3-insts,+ci-insts,+clusters,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+fp8e5m3-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+prng-inst,+setprio-inc-wg-inst,+tanh-insts,+tensor-cvt-lut-insts,+transpose-load-f4f6-insts,+vmem-pref-insts,+wavefrontsize32" -// GFX1251: "target-features"="+16-bit-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-fmin-fmax-global-f32,+atomic-fmin-fmax-global-f64,+atomic-global-pk-add-bf16-inst,+bf16-cvt-insts,+bf16-pk-insts,+bf16-trans-insts,+bitop3-insts,+ci-insts,+clusters,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+fp8e5m3-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+prng-inst,+setprio-inc-wg-inst,+tanh-insts,+tensor-cvt-lut-insts,+transpose-load-f4f6-insts,+vmem-pref-insts,+wavefrontsize32" +// GFX1250: "target-features"="+16-bit-insts,+add-min-max-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-fmin-fmax-global-f32,+atomic-fmin-fmax-global-f64,+atomic-global-pk-add-bf16-inst,+bf16-cvt-insts,+bf16-pk-insts,+bf16-trans-insts,+bitop3-insts,+ci-insts,+clusters,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+fp8e5m3-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+pk-add-min-max-insts,+prng-inst,+setprio-inc-wg-inst,+tanh-insts,+tensor-cvt-lut-insts,+transpose-load-f4f6-insts,+vmem-pref-insts,+wavefrontsize32" +// GFX1251: "target-features"="+16-bit-insts,+add-min-max-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-fmin-fmax-global-f32,+atomic-fmin-fmax-global-f64,+atomic-global-pk-add-bf16-inst,+bf16-cvt-insts,+bf16-pk-insts,+bf16-trans-insts,+bitop3-insts,+ci-insts,+clusters,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+fp8e5m3-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+pk-add-min-max-insts,+prng-inst,+setprio-inc-wg-inst,+tanh-insts,+tensor-cvt-lut-insts,+transpose-load-f4f6-insts,+vmem-pref-insts,+wavefrontsize32" // GFX1103-W64: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+atomic-fmin-fmax-global-f32,+ci-insts,+dl-insts,+dot10-insts,+dot12-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize64" diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl index b6b475a7..e4a5fe9 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl @@ -21,6 +21,7 @@ typedef float __attribute__((ext_vector_type(8))) float8; typedef float __attribute__((ext_vector_type(16))) float16; typedef float __attribute__((ext_vector_type(32))) float32; typedef short __attribute__((ext_vector_type(2))) short2; +typedef unsigned short __attribute__((ext_vector_type(2))) ushort2; // CHECK-LABEL: @test_setprio_inc_wg( // CHECK-NEXT: entry: @@ -1718,3 +1719,111 @@ void test_cvt_f32_fp8_e5m3(global int* out, int a) *out = __builtin_amdgcn_cvt_f32_fp8_e5m3(a, 2); *out = __builtin_amdgcn_cvt_f32_fp8_e5m3(a, 3); } + +// CHECK-LABEL: @test_add_min_max( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5) +// CHECK-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5) +// CHECK-NEXT: [[B_ADDR:%.*]] = alloca i32, align 4, addrspace(5) +// CHECK-NEXT: [[C_ADDR:%.*]] = alloca i32, align 4, addrspace(5) +// CHECK-NEXT: [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr +// CHECK-NEXT: [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr +// CHECK-NEXT: [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[B_ADDR]] to ptr +// CHECK-NEXT: [[C_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[C_ADDR]] to ptr +// CHECK-NEXT: store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4 +// CHECK-NEXT: store i32 [[B:%.*]], ptr [[B_ADDR_ASCAST]], align 4 +// CHECK-NEXT: store i32 [[C:%.*]], ptr [[C_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[B_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[C_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.add.max.i32(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]], i1 false) +// CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(1) [[TMP4]], align 4 +// CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[B_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[C_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.add.max.u32(i32 [[TMP5]], i32 [[TMP6]], i32 [[TMP7]], i1 true) +// CHECK-NEXT: [[TMP9:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i32 [[TMP8]], ptr addrspace(1) [[TMP9]], align 4 +// CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[B_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[C_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.amdgcn.add.min.i32(i32 [[TMP10]], i32 [[TMP11]], i32 [[TMP12]], i1 false) +// CHECK-NEXT: [[TMP14:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[TMP14]], align 4 +// CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[B_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[C_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.amdgcn.add.min.u32(i32 [[TMP15]], i32 [[TMP16]], i32 [[TMP17]], i1 true) +// CHECK-NEXT: [[TMP19:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i32 [[TMP18]], ptr addrspace(1) [[TMP19]], align 4 +// CHECK-NEXT: ret void +// +void test_add_min_max(global int *out, int a, int b, int c) +{ + *out = __builtin_amdgcn_add_max_i32(a, b, c, false); + *out = __builtin_amdgcn_add_max_u32(a, b, c, true); + *out = __builtin_amdgcn_add_min_i32(a, b, c, false); + *out = __builtin_amdgcn_add_min_u32(a, b, c, true); +} + +// CHECK-LABEL: @test_pk_add_min_max( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5) +// CHECK-NEXT: [[UOUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5) +// CHECK-NEXT: [[A_ADDR:%.*]] = alloca <2 x i16>, align 4, addrspace(5) +// CHECK-NEXT: [[B_ADDR:%.*]] = alloca <2 x i16>, align 4, addrspace(5) +// CHECK-NEXT: [[C_ADDR:%.*]] = alloca <2 x i16>, align 4, addrspace(5) +// CHECK-NEXT: [[UA_ADDR:%.*]] = alloca <2 x i16>, align 4, addrspace(5) +// CHECK-NEXT: [[UB_ADDR:%.*]] = alloca <2 x i16>, align 4, addrspace(5) +// CHECK-NEXT: [[UC_ADDR:%.*]] = alloca <2 x i16>, align 4, addrspace(5) +// CHECK-NEXT: [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr +// CHECK-NEXT: [[UOUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[UOUT_ADDR]] to ptr +// CHECK-NEXT: [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr +// CHECK-NEXT: [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[B_ADDR]] to ptr +// CHECK-NEXT: [[C_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[C_ADDR]] to ptr +// CHECK-NEXT: [[UA_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[UA_ADDR]] to ptr +// CHECK-NEXT: [[UB_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[UB_ADDR]] to ptr +// CHECK-NEXT: [[UC_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[UC_ADDR]] to ptr +// CHECK-NEXT: store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store ptr addrspace(1) [[UOUT:%.*]], ptr [[UOUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store <2 x i16> [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4 +// CHECK-NEXT: store <2 x i16> [[B:%.*]], ptr [[B_ADDR_ASCAST]], align 4 +// CHECK-NEXT: store <2 x i16> [[C:%.*]], ptr [[C_ADDR_ASCAST]], align 4 +// CHECK-NEXT: store <2 x i16> [[UA:%.*]], ptr [[UA_ADDR_ASCAST]], align 4 +// CHECK-NEXT: store <2 x i16> [[UB:%.*]], ptr [[UB_ADDR_ASCAST]], align 4 +// CHECK-NEXT: store <2 x i16> [[UC:%.*]], ptr [[UC_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load <2 x i16>, ptr [[A_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load <2 x i16>, ptr [[B_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP2:%.*]] = load <2 x i16>, ptr [[C_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i16> @llvm.amdgcn.pk.add.max.i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]], <2 x i16> [[TMP2]], i1 false) +// CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store <2 x i16> [[TMP3]], ptr addrspace(1) [[TMP4]], align 4 +// CHECK-NEXT: [[TMP5:%.*]] = load <2 x i16>, ptr [[UA_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP6:%.*]] = load <2 x i16>, ptr [[UB_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP7:%.*]] = load <2 x i16>, ptr [[UC_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP8:%.*]] = call <2 x i16> @llvm.amdgcn.pk.add.max.u16(<2 x i16> [[TMP5]], <2 x i16> [[TMP6]], <2 x i16> [[TMP7]], i1 true) +// CHECK-NEXT: [[TMP9:%.*]] = load ptr addrspace(1), ptr [[UOUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store <2 x i16> [[TMP8]], ptr addrspace(1) [[TMP9]], align 4 +// CHECK-NEXT: [[TMP10:%.*]] = load <2 x i16>, ptr [[A_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP11:%.*]] = load <2 x i16>, ptr [[B_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP12:%.*]] = load <2 x i16>, ptr [[C_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP13:%.*]] = call <2 x i16> @llvm.amdgcn.pk.add.min.i16(<2 x i16> [[TMP10]], <2 x i16> [[TMP11]], <2 x i16> [[TMP12]], i1 false) +// CHECK-NEXT: [[TMP14:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store <2 x i16> [[TMP13]], ptr addrspace(1) [[TMP14]], align 4 +// CHECK-NEXT: [[TMP15:%.*]] = load <2 x i16>, ptr [[UA_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP16:%.*]] = load <2 x i16>, ptr [[UB_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = load <2 x i16>, ptr [[UC_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = call <2 x i16> @llvm.amdgcn.pk.add.min.u16(<2 x i16> [[TMP15]], <2 x i16> [[TMP16]], <2 x i16> [[TMP17]], i1 true) +// CHECK-NEXT: [[TMP19:%.*]] = load ptr addrspace(1), ptr [[UOUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store <2 x i16> [[TMP18]], ptr addrspace(1) [[TMP19]], align 4 +// CHECK-NEXT: ret void +// +void test_pk_add_min_max(global short2 *out, global ushort2 *uout, short2 a, short2 b, short2 c, ushort2 ua, ushort2 ub, ushort2 uc) +{ + *out = __builtin_amdgcn_pk_add_max_i16(a, b, c, false); + *uout = __builtin_amdgcn_pk_add_max_u16(ua, ub, uc, true); + *out = __builtin_amdgcn_pk_add_min_i16(a, b, c, false); + *uout = __builtin_amdgcn_pk_add_min_u16(ua, ub, uc, true); +} diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl index 3cea47b..da6a03b 100644 --- a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl +++ b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl @@ -15,6 +15,8 @@ typedef half __attribute__((ext_vector_type(32))) half32; typedef float __attribute__((ext_vector_type(8))) float8; typedef float __attribute__((ext_vector_type(16))) float16; typedef float __attribute__((ext_vector_type(32))) float32; +typedef short __attribute__((ext_vector_type(2))) short2; +typedef unsigned short __attribute__((ext_vector_type(2))) ushort2; typedef int v4i __attribute__((ext_vector_type(4))); typedef int v8i __attribute__((ext_vector_type(8))); @@ -165,3 +167,19 @@ void test_cvt_f32_fp8_e5m3(global int* out, int a) { *out = __builtin_amdgcn_cvt_f32_fp8_e5m3(a, a); // expected-error {{'__builtin_amdgcn_cvt_f32_fp8_e5m3' must be a constant integer}} } + +void test_add_min_max(global int *out, int a, int b, int c, bool clamp) +{ + *out = __builtin_amdgcn_add_max_i32(a, b, c, clamp); // expected-error {{'__builtin_amdgcn_add_max_i32' must be a constant integer}} + *out = __builtin_amdgcn_add_max_u32(a, b, c, clamp); // expected-error {{'__builtin_amdgcn_add_max_u32' must be a constant integer}} + *out = __builtin_amdgcn_add_min_i32(a, b, c, clamp); // expected-error {{'__builtin_amdgcn_add_min_i32' must be a constant integer}} + *out = __builtin_amdgcn_add_min_u32(a, b, c, clamp); // expected-error {{'__builtin_amdgcn_add_min_u32' must be a constant integer}} +} + +void test_pk_add_min_max(global short2 *out, global ushort2 *uout, short2 a, short2 b, short2 c, ushort2 ua, ushort2 ub, ushort2 uc, bool clamp) +{ + *out = __builtin_amdgcn_pk_add_max_i16(a, b, c, clamp); // expected-error {{'__builtin_amdgcn_pk_add_max_i16' must be a constant integer}} + *uout = __builtin_amdgcn_pk_add_max_u16(ua, ub, uc, clamp); // expected-error {{'__builtin_amdgcn_pk_add_max_u16' must be a constant integer}} + *out = __builtin_amdgcn_pk_add_min_i16(a, b, c, clamp); // expected-error {{'__builtin_amdgcn_pk_add_min_i16' must be a constant integer}} + *uout = __builtin_amdgcn_pk_add_min_u16(ua, ub, uc, clamp); // expected-error {{'__builtin_amdgcn_pk_add_min_u16' must be a constant integer}} +} diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250.cl index d7045cd..e53beae 100644 --- a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250.cl +++ b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250.cl @@ -3,9 +3,21 @@ typedef unsigned int uint; typedef unsigned short int ushort; +typedef short __attribute__((ext_vector_type(2))) short2; +typedef unsigned short __attribute__((ext_vector_type(2))) ushort2; -void test(global uint* out, uint a, uint b, uint c) { +void test(global int* out, global short2 *s2out, global ushort2 *us2out, + uint a, uint b, uint c, short2 s2a, short2 s2b, short2 s2c, + ushort2 us2a, ushort2 us2b, ushort2 us2c) { __builtin_amdgcn_s_setprio_inc_wg(1); // expected-error {{'__builtin_amdgcn_s_setprio_inc_wg' needs target feature setprio-inc-wg-inst}} *out = __builtin_amdgcn_bitop3_b32(a, b, c, 1); // expected-error {{'__builtin_amdgcn_bitop3_b32' needs target feature bitop3-insts}} *out = __builtin_amdgcn_bitop3_b16((ushort)a, (ushort)b, (ushort)c, 1); // expected-error {{'__builtin_amdgcn_bitop3_b16' needs target feature bitop3-insts}} + *out = __builtin_amdgcn_add_max_i32(a, b, c, false); // expected-error {{'__builtin_amdgcn_add_max_i32' needs target feature add-min-max-insts}} + *out = __builtin_amdgcn_add_max_u32(a, b, c, true); // expected-error {{'__builtin_amdgcn_add_max_u32' needs target feature add-min-max-insts}} + *out = __builtin_amdgcn_add_min_i32(a, b, c, false); // expected-error {{'__builtin_amdgcn_add_min_i32' needs target feature add-min-max-insts}} + *out = __builtin_amdgcn_add_min_u32(a, b, c, true); // expected-error {{'__builtin_amdgcn_add_min_u32' needs target feature add-min-max-insts}} + *s2out = __builtin_amdgcn_pk_add_max_i16(s2a, s2b, s2c, false); // expected-error {{'__builtin_amdgcn_pk_add_max_i16' needs target feature pk-add-min-max-insts}} + *us2out = __builtin_amdgcn_pk_add_max_u16(us2a, us2b, us2c, true); // expected-error {{'__builtin_amdgcn_pk_add_max_u16' needs target feature pk-add-min-max-insts}} + *s2out = __builtin_amdgcn_pk_add_min_i16(s2a, s2b, s2c, false); // expected-error {{'__builtin_amdgcn_pk_add_min_i16' needs target feature pk-add-min-max-insts}} + *us2out = __builtin_amdgcn_pk_add_min_u16(us2a, us2b, us2c, true); // expected-error {{'__builtin_amdgcn_pk_add_min_u16' needs target feature pk-add-min-max-insts}} } diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index e41f4eb..c11a348 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -106,6 +106,7 @@ static constexpr bool DoRoundTripDefault = false; #endif static bool RoundTripArgs = DoRoundTripDefault; +static bool VerbatimArgs = false; static void ParseArgs(int argc, char **argv) { ScanDepsOptTable Tbl; @@ -239,6 +240,8 @@ static void ParseArgs(int argc, char **argv) { RoundTripArgs = Args.hasArg(OPT_round_trip_args); + VerbatimArgs = Args.hasArg(OPT_verbatim_args); + if (const llvm::opt::Arg *A = Args.getLastArgNoClaim(OPT_DASH_DASH)) CommandLine.assign(A->getValues().begin(), A->getValues().end()); } @@ -883,14 +886,16 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { llvm::cl::PrintOptionValues(); - // Expand response files in advance, so that we can "see" all the arguments - // when adjusting below. - Compilations = expandResponseFiles(std::move(Compilations), - llvm::vfs::getRealFileSystem()); + if (!VerbatimArgs) { + // Expand response files in advance, so that we can "see" all the arguments + // when adjusting below. + Compilations = expandResponseFiles(std::move(Compilations), + llvm::vfs::getRealFileSystem()); - Compilations = inferTargetAndDriverMode(std::move(Compilations)); + Compilations = inferTargetAndDriverMode(std::move(Compilations)); - Compilations = inferToolLocation(std::move(Compilations)); + Compilations = inferToolLocation(std::move(Compilations)); + } // The command options are rewritten to run Clang in preprocessor only mode. auto AdjustingCompilations = @@ -898,7 +903,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { std::move(Compilations)); ResourceDirectoryCache ResourceDirCache; - AdjustingCompilations->appendArgumentsAdjuster( + auto ArgsAdjuster = [&ResourceDirCache](const tooling::CommandLineArguments &Args, StringRef FileName) { std::string LastO; @@ -960,7 +965,10 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { } AdjustedArgs.insert(AdjustedArgs.end(), FlagsEnd, Args.end()); return AdjustedArgs; - }); + }; + + if (!VerbatimArgs) + AdjustingCompilations->appendArgumentsAdjuster(ArgsAdjuster); SharedStream Errs(llvm::errs()); diff --git a/clang/tools/clang-scan-deps/Opts.td b/clang/tools/clang-scan-deps/Opts.td index 03011f9..7a63b18 100644 --- a/clang/tools/clang-scan-deps/Opts.td +++ b/clang/tools/clang-scan-deps/Opts.td @@ -44,4 +44,6 @@ def verbose : F<"v", "Use verbose output">; def round_trip_args : F<"round-trip-args", "verify that command-line arguments are canonical by parsing and re-serializing">; +def verbatim_args : F<"verbatim-args", "Pass commands to the scanner verbatim without adjustments">; + def DASH_DASH : Option<["--"], "", KIND_REMAINING_ARGS>; diff --git a/clang/unittests/Analysis/FlowSensitive/MockHeaders.cpp b/clang/unittests/Analysis/FlowSensitive/MockHeaders.cpp index d875542..2e528ed 100644 --- a/clang/unittests/Analysis/FlowSensitive/MockHeaders.cpp +++ b/clang/unittests/Analysis/FlowSensitive/MockHeaders.cpp @@ -1356,7 +1356,7 @@ bool operator==(const Status &lhs, const Status &rhs); bool operator!=(const Status &lhs, const Status &rhs); Status OkStatus(); -Status InvalidArgumentError(char *); +Status InvalidArgumentError(const char *); #endif // STATUS_H )cc"; diff --git a/clang/unittests/Analysis/FlowSensitive/UncheckedStatusOrAccessModelTestFixture.cpp b/clang/unittests/Analysis/FlowSensitive/UncheckedStatusOrAccessModelTestFixture.cpp index cae9265..4bb09d3 100644 --- a/clang/unittests/Analysis/FlowSensitive/UncheckedStatusOrAccessModelTestFixture.cpp +++ b/clang/unittests/Analysis/FlowSensitive/UncheckedStatusOrAccessModelTestFixture.cpp @@ -2614,6 +2614,263 @@ TEST_P(UncheckedStatusOrAccessModelTest, StatusUpdate) { )cc"); } +TEST_P(UncheckedStatusOrAccessModelTest, EqualityCheck) { + ExpectDiagnosticsFor( + R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (x == y) + y.value(); + else + y.value(); // [[unsafe]] + } + } + )cc"); + ExpectDiagnosticsFor( + R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (y == x) + y.value(); + else + y.value(); // [[unsafe]] + } + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (x != y) + y.value(); // [[unsafe]] + else + y.value(); + } + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (y != x) + y.value(); // [[unsafe]] + else + y.value(); + } + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (!(x == y)) + y.value(); // [[unsafe]] + else + y.value(); + } + } + )cc"); + ExpectDiagnosticsFor( + R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (!(x != y)) + y.value(); + else + y.value(); // [[unsafe]] + } + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x == y) + if (x.ok()) y.value(); + } + )cc"); + ExpectDiagnosticsFor( + R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (x.status() == y.status()) + y.value(); + else + y.value(); // [[unsafe]] + } + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (x.status() != y.status()) + y.value(); // [[unsafe]] + else + y.value(); + } + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT sor) { + if (sor.status() == absl::OkStatus()) + sor.value(); + else + sor.value(); // [[unsafe]] + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT sor) { + if (sor.status() != absl::OkStatus()) + sor.value(); // [[unsafe]] + else + sor.value(); + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT sor) { + if (sor.status() != absl::InvalidArgumentError("oh no")) + sor.value(); // [[unsafe]] + else + sor.value(); // [[unsafe]] + } + )cc"); + ExpectDiagnosticsFor( + R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (x.ok() == y.ok()) + y.value(); + else + y.value(); // [[unsafe]] + } + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (x.ok() != y.ok()) + y.value(); // [[unsafe]] + else + y.value(); + } + } + )cc"); + ExpectDiagnosticsFor( + R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (x.status().ok() == y.status().ok()) + y.value(); + else + y.value(); // [[unsafe]] + } + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (x.status().ok() != y.status().ok()) + y.value(); // [[unsafe]] + else + y.value(); + } + } + )cc"); + ExpectDiagnosticsFor( + R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (x.status().ok() == y.ok()) + y.value(); + else + y.value(); // [[unsafe]] + } + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT x, STATUSOR_INT y) { + if (x.ok()) { + if (x.status().ok() != y.ok()) + y.value(); // [[unsafe]] + else + y.value(); + } + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(bool b, STATUSOR_INT sor) { + if (sor.ok() == b) { + if (b) sor.value(); + } + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT sor) { + if (sor.ok() == true) sor.value(); + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(STATUSOR_INT sor) { + if (sor.ok() == false) sor.value(); // [[unsafe]] + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(bool b) { + STATUSOR_INT sor1; + STATUSOR_INT sor2 = Make<STATUSOR_INT>(); + if (sor1 == sor2) sor2.value(); // [[unsafe]] + } + )cc"); + ExpectDiagnosticsFor(R"cc( +#include "unchecked_statusor_access_test_defs.h" + + void target(bool b) { + STATUSOR_INT sor1 = Make<STATUSOR_INT>(); + STATUSOR_INT sor2; + if (sor1 == sor2) sor1.value(); // [[unsafe]] + } + )cc"); +} + } // namespace std::string diff --git a/clang/unittests/CIR/PointerLikeTest.cpp b/clang/unittests/CIR/PointerLikeTest.cpp index 22690f2..b3dfba7 100644 --- a/clang/unittests/CIR/PointerLikeTest.cpp +++ b/clang/unittests/CIR/PointerLikeTest.cpp @@ -76,7 +76,7 @@ protected: EXPECT_EQ(pltTy.getElementType(), ty); OwningOpRef<cir::AllocaOp> varPtrOp = - b.create<cir::AllocaOp>(loc, ptrTy, ty, "", getAlignOne(&context)); + cir::AllocaOp::create(b, loc, ptrTy, ty, "", getAlignOne(&context)); mlir::Value val = varPtrOp.get(); mlir::acc::VariableTypeCategory typeCategory = pltTy.getPointeeTypeCategory( @@ -110,7 +110,7 @@ protected: // Create an alloca for the array OwningOpRef<cir::AllocaOp> varPtrOp = - b.create<cir::AllocaOp>(loc, ptrTy, arrTy, "", getAlignOne(&context)); + cir::AllocaOp::create(b, loc, ptrTy, arrTy, "", getAlignOne(&context)); // Verify that the type category is array. mlir::Value val = varPtrOp.get(); @@ -121,8 +121,8 @@ protected: // Create an array-to-pointer decay cast. mlir::Type ptrToElemTy = cir::PointerType::get(ty); - OwningOpRef<cir::CastOp> decayPtr = b.create<cir::CastOp>( - loc, ptrToElemTy, cir::CastKind::array_to_ptrdecay, val); + OwningOpRef<cir::CastOp> decayPtr = cir::CastOp::create( + b, loc, ptrToElemTy, cir::CastKind::array_to_ptrdecay, val); mlir::Value decayVal = decayPtr.get(); // Verify that we still get the expected element type. @@ -141,9 +141,9 @@ protected: // Create an element access. mlir::Type i32Ty = cir::IntType::get(&context, 32, true); mlir::Value index = - b.create<cir::ConstantOp>(loc, cir::IntAttr::get(i32Ty, 2)); + cir::ConstantOp::create(b, loc, cir::IntAttr::get(i32Ty, 2)); OwningOpRef<cir::PtrStrideOp> accessPtr = - b.create<cir::PtrStrideOp>(loc, ptrToElemTy, decayVal, index); + cir::PtrStrideOp::create(b, loc, ptrToElemTy, decayVal, index); mlir::Value accessVal = accessPtr.get(); // Verify that we still get the expected element type. @@ -175,8 +175,8 @@ protected: EXPECT_EQ(pltTy.getElementType(), structTy); // Create an alloca for the array - OwningOpRef<cir::AllocaOp> varPtrOp = b.create<cir::AllocaOp>( - loc, ptrTy, structTy, "", getAlignOne(&context)); + OwningOpRef<cir::AllocaOp> varPtrOp = cir::AllocaOp::create( + b, loc, ptrTy, structTy, "", getAlignOne(&context)); // Verify that the type category is composite. mlir::Value val = varPtrOp.get(); @@ -186,8 +186,8 @@ protected: EXPECT_EQ(typeCategory, mlir::acc::VariableTypeCategory::composite); // Access the first element of the structure. - OwningOpRef<cir::GetMemberOp> access1 = b.create<cir::GetMemberOp>( - loc, cir::PointerType::get(ty1), val, b.getStringAttr("f1"), 0); + OwningOpRef<cir::GetMemberOp> access1 = cir::GetMemberOp::create( + b, loc, cir::PointerType::get(ty1), val, "f1", 0u); mlir::Value accessVal1 = access1.get(); // Verify that we get the expected element type. @@ -204,8 +204,8 @@ protected: EXPECT_EQ(access1TypeCategory, mlir::acc::VariableTypeCategory::composite); // Access the second element of the structure. - OwningOpRef<cir::GetMemberOp> access2 = b.create<cir::GetMemberOp>( - loc, cir::PointerType::get(ty2), val, b.getStringAttr("f2"), 1); + OwningOpRef<cir::GetMemberOp> access2 = cir::GetMemberOp::create( + b, loc, cir::PointerType::get(ty2), val, "f2", 1u); mlir::Value accessVal2 = access2.get(); // Verify that we get the expected element type. @@ -252,17 +252,17 @@ protected: mlir::Type structPptrTy = cir::PointerType::get(structTy); // Create an alloca for the struct. - OwningOpRef<cir::AllocaOp> varPtrOp = b.create<cir::AllocaOp>( - loc, structPptrTy, structTy, "S", getAlignOne(&context)); + OwningOpRef<cir::AllocaOp> varPtrOp = cir::AllocaOp::create( + b, loc, structPptrTy, structTy, "S", getAlignOne(&context)); mlir::Value val = varPtrOp.get(); // Get a pointer to the second member. - OwningOpRef<cir::GetMemberOp> access = b.create<cir::GetMemberOp>( - loc, cir::PointerType::get(ptrTy), val, b.getStringAttr("f2"), 1); + OwningOpRef<cir::GetMemberOp> access = cir::GetMemberOp::create( + b, loc, cir::PointerType::get(ptrTy), val, b.getStringAttr("f2"), 1); mlir::Value accessVal = access.get(); // Load the value of the second member. This is the pointer we want to test. - OwningOpRef<cir::LoadOp> loadOp = b.create<cir::LoadOp>(loc, accessVal); + OwningOpRef<cir::LoadOp> loadOp = cir::LoadOp::create(b, loc, accessVal); mlir::Value loadVal = loadOp.get(); // Verify that the type category is the expected type category. diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index f363738..ca99940 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -1129,6 +1129,11 @@ TEST_F(TokenAnnotatorTest, UnderstandsOverloadedOperators) { ASSERT_EQ(Tokens.size(), 7u) << Tokens; // Not TT_FunctionDeclarationName. EXPECT_TOKEN(Tokens[3], tok::kw_operator, TT_Unknown); + + Tokens = annotate("SomeAPI::operator()();"); + ASSERT_EQ(Tokens.size(), 9u) << Tokens; + // Not TT_FunctionDeclarationName. + EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown); } TEST_F(TokenAnnotatorTest, OverloadedOperatorInTemplate) { diff --git a/compiler-rt/lib/builtins/gcc_personality_v0.c b/compiler-rt/lib/builtins/gcc_personality_v0.c index 3ed17fa..6d92a7b 100644 --- a/compiler-rt/lib/builtins/gcc_personality_v0.c +++ b/compiler-rt/lib/builtins/gcc_personality_v0.c @@ -30,6 +30,10 @@ EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, void *, PCONTEXT, _Unwind_Personality_Fn); #endif +#ifndef __has_feature +#define __has_feature(__feature) 0 +#endif + #if __has_feature(ptrauth_calls) #include <ptrauth.h> diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index 29eedfb..1ae8bbe 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -989,9 +989,18 @@ static constexpr IntrinsicHandler handlers[]{ {"mask", asBox, handleDynamicOptional}}}, /*isElemental=*/false}, {"syncthreads", &I::genSyncThreads, {}, /*isElemental=*/false}, - {"syncthreads_and", &I::genSyncThreadsAnd, {}, /*isElemental=*/false}, - {"syncthreads_count", &I::genSyncThreadsCount, {}, /*isElemental=*/false}, - {"syncthreads_or", &I::genSyncThreadsOr, {}, /*isElemental=*/false}, + {"syncthreads_and_i4", &I::genSyncThreadsAnd, {}, /*isElemental=*/false}, + {"syncthreads_and_l4", &I::genSyncThreadsAnd, {}, /*isElemental=*/false}, + {"syncthreads_count_i4", + &I::genSyncThreadsCount, + {}, + /*isElemental=*/false}, + {"syncthreads_count_l4", + &I::genSyncThreadsCount, + {}, + /*isElemental=*/false}, + {"syncthreads_or_i4", &I::genSyncThreadsOr, {}, /*isElemental=*/false}, + {"syncthreads_or_l4", &I::genSyncThreadsOr, {}, /*isElemental=*/false}, {"syncwarp", &I::genSyncWarp, {}, /*isElemental=*/false}, {"system", &I::genSystem, @@ -8965,10 +8974,12 @@ IntrinsicLibrary::genSyncThreadsAnd(mlir::Type resultType, llvm::ArrayRef<mlir::Value> args) { constexpr llvm::StringLiteral funcName = "llvm.nvvm.barrier0.and"; mlir::MLIRContext *context = builder.getContext(); + mlir::Type i32 = builder.getI32Type(); mlir::FunctionType ftype = - mlir::FunctionType::get(context, {resultType}, {args[0].getType()}); + mlir::FunctionType::get(context, {resultType}, {i32}); auto funcOp = builder.createFunction(loc, funcName, ftype); - return fir::CallOp::create(builder, loc, funcOp, args).getResult(0); + mlir::Value arg = builder.createConvert(loc, i32, args[0]); + return fir::CallOp::create(builder, loc, funcOp, {arg}).getResult(0); } // SYNCTHREADS_COUNT @@ -8977,10 +8988,12 @@ IntrinsicLibrary::genSyncThreadsCount(mlir::Type resultType, llvm::ArrayRef<mlir::Value> args) { constexpr llvm::StringLiteral funcName = "llvm.nvvm.barrier0.popc"; mlir::MLIRContext *context = builder.getContext(); + mlir::Type i32 = builder.getI32Type(); mlir::FunctionType ftype = - mlir::FunctionType::get(context, {resultType}, {args[0].getType()}); + mlir::FunctionType::get(context, {resultType}, {i32}); auto funcOp = builder.createFunction(loc, funcName, ftype); - return fir::CallOp::create(builder, loc, funcOp, args).getResult(0); + mlir::Value arg = builder.createConvert(loc, i32, args[0]); + return fir::CallOp::create(builder, loc, funcOp, {arg}).getResult(0); } // SYNCTHREADS_OR @@ -8989,10 +9002,12 @@ IntrinsicLibrary::genSyncThreadsOr(mlir::Type resultType, llvm::ArrayRef<mlir::Value> args) { constexpr llvm::StringLiteral funcName = "llvm.nvvm.barrier0.or"; mlir::MLIRContext *context = builder.getContext(); + mlir::Type i32 = builder.getI32Type(); mlir::FunctionType ftype = - mlir::FunctionType::get(context, {resultType}, {args[0].getType()}); + mlir::FunctionType::get(context, {resultType}, {i32}); auto funcOp = builder.createFunction(loc, funcName, ftype); - return fir::CallOp::create(builder, loc, funcOp, args).getResult(0); + mlir::Value arg = builder.createConvert(loc, i32, args[0]); + return fir::CallOp::create(builder, loc, funcOp, {arg}).getResult(0); } // SYNCWARP diff --git a/flang/module/cudadevice.f90 b/flang/module/cudadevice.f90 index 22df9cd..5182950 100644 --- a/flang/module/cudadevice.f90 +++ b/flang/module/cudadevice.f90 @@ -21,23 +21,32 @@ implicit none procedure :: syncthreads end interface - interface - attributes(device) integer function syncthreads_and(value) - integer, value :: value + interface syncthreads_and + attributes(device) integer function syncthreads_and_i4(value) + integer(4), value :: value end function - end interface + attributes(device) integer function syncthreads_and_l4(value) + logical(4), value :: value + end function + end interface syncthreads_and - interface - attributes(device) integer function syncthreads_count(value) - integer, value :: value + interface syncthreads_count + attributes(device) integer function syncthreads_count_i4(value) + integer(4), value :: value end function - end interface + attributes(device) integer function syncthreads_count_l4(value) + logical(4), value :: value + end function + end interface syncthreads_count - interface - attributes(device) integer function syncthreads_or(value) - integer, value :: value + interface syncthreads_or + attributes(device) integer function syncthreads_or_i4(value) + integer(4), value :: value end function - end interface + attributes(device) integer function syncthreads_or_l4(value) + logical(4), value :: value + end function + end interface syncthreads_or interface attributes(device) subroutine syncwarp(mask) diff --git a/flang/test/Lower/CUDA/cuda-device-proc.cuf b/flang/test/Lower/CUDA/cuda-device-proc.cuf index 29c348c..7d6caf5 100644 --- a/flang/test/Lower/CUDA/cuda-device-proc.cuf +++ b/flang/test/Lower/CUDA/cuda-device-proc.cuf @@ -12,17 +12,23 @@ attributes(global) subroutine devsub() integer(8) :: al integer(8) :: time integer :: smalltime - integer(4) :: res + integer(4) :: res, offset integer(8) :: resl + integer :: tid + tid = threadIdx%x + call syncthreads() call syncwarp(1) call threadfence() call threadfence_block() call threadfence_system() ret = syncthreads_and(1) + res = syncthreads_and(tid > offset) ret = syncthreads_count(1) + ret = syncthreads_count(tid > offset) ret = syncthreads_or(1) + ret = syncthreads_or(tid > offset) ai = atomicadd(ai, 1_4) al = atomicadd(al, 1_8) @@ -100,9 +106,24 @@ end ! CHECK: fir.call @llvm.nvvm.membar.gl() fastmath<contract> : () -> () ! CHECK: fir.call @llvm.nvvm.membar.cta() fastmath<contract> : () -> () ! CHECK: fir.call @llvm.nvvm.membar.sys() fastmath<contract> : () -> () -! CHECK: %{{.*}} = fir.call @llvm.nvvm.barrier0.and(%c1_i32_0) fastmath<contract> : (i32) -> i32 -! CHECK: %{{.*}} = fir.call @llvm.nvvm.barrier0.popc(%c1_i32_1) fastmath<contract> : (i32) -> i32 -! CHECK: %{{.*}} = fir.call @llvm.nvvm.barrier0.or(%c1_i32_2) fastmath<contract> : (i32) -> i32 +! CHECK: %{{.*}} = fir.call @llvm.nvvm.barrier0.and(%c1{{.*}}) fastmath<contract> : (i32) -> i32 +! CHECK: %[[A:.*]] = fir.load %{{.*}} : !fir.ref<i32> +! CHECK: %[[B:.*]] = fir.load %{{.*}} : !fir.ref<i32> +! CHECK: %[[CMP:.*]] = arith.cmpi sgt, %[[A]], %[[B]] : i32 +! CHECK: %[[CONV:.*]] = fir.convert %[[CMP]] : (i1) -> i32 +! CHECK: %{{.*}} = fir.call @llvm.nvvm.barrier0.and(%[[CONV]]) +! CHECK: %{{.*}} = fir.call @llvm.nvvm.barrier0.popc(%c1{{.*}}) fastmath<contract> : (i32) -> i32 +! CHECK: %[[A:.*]] = fir.load %{{.*}} : !fir.ref<i32> +! CHECK: %[[B:.*]] = fir.load %{{.*}} : !fir.ref<i32> +! CHECK: %[[CMP:.*]] = arith.cmpi sgt, %[[A]], %[[B]] : i32 +! CHECK: %[[CONV:.*]] = fir.convert %[[CMP]] : (i1) -> i32 +! CHECK: %{{.*}} = fir.call @llvm.nvvm.barrier0.popc(%[[CONV]]) fastmath<contract> : (i32) -> i32 +! CHECK: %{{.*}} = fir.call @llvm.nvvm.barrier0.or(%c1{{.*}}) fastmath<contract> : (i32) -> i32 +! CHECK: %[[A:.*]] = fir.load %{{.*}} : !fir.ref<i32> +! CHECK: %[[B:.*]] = fir.load %{{.*}} : !fir.ref<i32> +! CHECK: %[[CMP:.*]] = arith.cmpi sgt, %[[A]], %[[B]] : i32 +! CHECK: %[[CONV:.*]] = fir.convert %[[CMP]] : (i1) -> i32 +! CHECK: %{{.*}} = fir.call @llvm.nvvm.barrier0.or(%[[CONV]]) fastmath<contract> : (i32) -> i32 ! CHECK: %{{.*}} = llvm.atomicrmw add %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32 ! CHECK: %{{.*}} = llvm.atomicrmw add %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i64 ! CHECK: %{{.*}} = llvm.atomicrmw fadd %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f32 diff --git a/libc/include/llvm-libc-macros/CMakeLists.txt b/libc/include/llvm-libc-macros/CMakeLists.txt index a33ff6c..b16337c 100644 --- a/libc/include/llvm-libc-macros/CMakeLists.txt +++ b/libc/include/llvm-libc-macros/CMakeLists.txt @@ -32,6 +32,12 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}) endif() add_macro_header( + annex_k_macros + HDR + annex-k-macros.h +) + +add_macro_header( assert_macros HDR assert-macros.h diff --git a/libc/include/llvm-libc-macros/annex-k-macros.h b/libc/include/llvm-libc-macros/annex-k-macros.h new file mode 100644 index 0000000..7cfb5c1 --- /dev/null +++ b/libc/include/llvm-libc-macros/annex-k-macros.h @@ -0,0 +1,27 @@ +//===-- Definition of macros to be used with Annex K functions ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_INCLUDE_LLVM_LIBC_MACROS_ANNEX_K_MACROS_H +#define LLVM_LIBC_INCLUDE_LLVM_LIBC_MACROS_ANNEX_K_MACROS_H + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ + (defined(__cplusplus) && __cplusplus >= 201703L) + +// TODO(bassiounix): Who should def this macro (clang vs libc)? Where? +// TODO(bassiounix): uncomment/move when Annex K is fully implemented. +// #define __STDC_LIB_EXT1__ 201112L + +#if defined(__STDC_WANT_LIB_EXT1__) && __STDC_WANT_LIB_EXT1__ == 1 + +#define LIBC_HAS_ANNEX_K + +#endif // defined(__STDC_WANT_LIB_EXT1__) && __STDC_WANT_LIB_EXT1__ == 1 + +#endif // (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || + // (defined(__cplusplus) && __cplusplus >= 201703L) +#endif // LLVM_LIBC_INCLUDE_LLVM_LIBC_MACROS_ANNEX_K_MACROS_H diff --git a/libcxx/include/__compare/strong_order.h b/libcxx/include/__compare/strong_order.h index 8c363b5..ba6de44 100644 --- a/libcxx/include/__compare/strong_order.h +++ b/libcxx/include/__compare/strong_order.h @@ -13,7 +13,6 @@ #include <__compare/compare_three_way.h> #include <__compare/ordering.h> #include <__config> -#include <__math/exponential_functions.h> #include <__math/traits.h> #include <__type_traits/conditional.h> #include <__type_traits/decay.h> @@ -53,38 +52,21 @@ struct __fn { template <class _Tp, class _Up, class _Dp = decay_t<_Tp>> requires is_same_v<_Dp, decay_t<_Up>> && is_floating_point_v<_Dp> _LIBCPP_HIDE_FROM_ABI static constexpr strong_ordering __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept { - if constexpr (numeric_limits<_Dp>::is_iec559 && sizeof(_Dp) == sizeof(int32_t)) { - int32_t __rx = std::bit_cast<int32_t>(__t); - int32_t __ry = std::bit_cast<int32_t>(__u); - __rx = (__rx < 0) ? (numeric_limits<int32_t>::min() - __rx - 1) : __rx; - __ry = (__ry < 0) ? (numeric_limits<int32_t>::min() - __ry - 1) : __ry; - return (__rx <=> __ry); - } else if constexpr (numeric_limits<_Dp>::is_iec559 && sizeof(_Dp) == sizeof(int64_t)) { - int64_t __rx = std::bit_cast<int64_t>(__t); - int64_t __ry = std::bit_cast<int64_t>(__u); - __rx = (__rx < 0) ? (numeric_limits<int64_t>::min() - __rx - 1) : __rx; - __ry = (__ry < 0) ? (numeric_limits<int64_t>::min() - __ry - 1) : __ry; + if constexpr (numeric_limits<_Dp>::is_iec559 && + (sizeof(_Dp) == sizeof(int32_t) || sizeof(_Dp) == sizeof(int64_t))) { + using _IntT = conditional_t<sizeof(_Dp) == sizeof(int32_t), int32_t, int64_t>; + _IntT __rx = std::bit_cast<_IntT>(__t); + _IntT __ry = std::bit_cast<_IntT>(__u); + __rx = (__rx < 0) ? (numeric_limits<_IntT>::min() - __rx - 1) : __rx; + __ry = (__ry < 0) ? (numeric_limits<_IntT>::min() - __ry - 1) : __ry; return (__rx <=> __ry); } else if (__t < __u) { return strong_ordering::less; } else if (__t > __u) { return strong_ordering::greater; } else if (__t == __u) { - if constexpr (numeric_limits<_Dp>::radix == 2) { - return __math::signbit(__u) <=> __math::signbit(__t); - } else { - // This is bullet 3 of the IEEE754 algorithm, relevant - // only for decimal floating-point; - // see https://stackoverflow.com/questions/69068075/ - if (__t == 0 || __math::isinf(__t)) { - return __math::signbit(__u) <=> __math::signbit(__t); - } else { - int __texp, __uexp; - (void)__math::frexp(__t, &__texp); - (void)__math::frexp(__u, &__uexp); - return (__t < 0) ? (__texp <=> __uexp) : (__uexp <=> __texp); - } - } + static_assert(numeric_limits<_Dp>::radix == 2, "floating point type with a radix other than 2?"); + return __math::signbit(__u) <=> __math::signbit(__t); } else { // They're unordered, so one of them must be a NAN. // The order is -QNAN, -SNAN, numbers, +SNAN, +QNAN. @@ -93,9 +75,9 @@ struct __fn { bool __t_is_negative = __math::signbit(__t); bool __u_is_negative = __math::signbit(__u); using _IntType = - conditional_t< sizeof(__t) == sizeof(int32_t), - int32_t, - conditional_t< sizeof(__t) == sizeof(int64_t), int64_t, void> >; + conditional_t<sizeof(__t) == sizeof(int32_t), + int32_t, + conditional_t<sizeof(__t) == sizeof(int64_t), int64_t, void>>; if constexpr (is_same_v<_IntType, void>) { static_assert(sizeof(_Dp) == 0, "std::strong_order is unimplemented for this floating-point type"); } else if (__t_is_nan && __u_is_nan) { diff --git a/libunwind/include/__libunwind_config.h b/libunwind/include/__libunwind_config.h index 343934e..980d11e 100644 --- a/libunwind/include/__libunwind_config.h +++ b/libunwind/include/__libunwind_config.h @@ -212,11 +212,13 @@ # define _LIBUNWIND_HIGHEST_DWARF_REGISTER 287 #endif // _LIBUNWIND_IS_NATIVE_ONLY -#if __has_feature(ptrauth_calls) && __has_feature(ptrauth_returns) -# define _LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING 1 -#elif __has_feature(ptrauth_calls) != __has_feature(ptrauth_returns) -# error "Either both or none of ptrauth_calls and ptrauth_returns "\ - "is allowed to be enabled" +#if defined(__has_feature) +# if __has_feature(ptrauth_calls) && __has_feature(ptrauth_returns) +# define _LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING 1 +# elif __has_feature(ptrauth_calls) != __has_feature(ptrauth_returns) +# error "Either both or none of ptrauth_calls and ptrauth_returns "\ + "is allowed to be enabled" +# endif #endif #endif // ____LIBUNWIND_CONFIG_H__ diff --git a/libunwind/src/UnwindRegistersRestore.S b/libunwind/src/UnwindRegistersRestore.S index 1ab4c43..198735f 100644 --- a/libunwind/src/UnwindRegistersRestore.S +++ b/libunwind/src/UnwindRegistersRestore.S @@ -634,6 +634,10 @@ Lnovec: #elif defined(__aarch64__) +#ifndef __has_feature +#define __has_feature(__feature) 0 +#endif + #if defined(__ARM_FEATURE_GCS_DEFAULT) .arch_extension gcs #endif diff --git a/libunwind/src/UnwindRegistersSave.S b/libunwind/src/UnwindRegistersSave.S index 31a177f..619a597 100644 --- a/libunwind/src/UnwindRegistersSave.S +++ b/libunwind/src/UnwindRegistersSave.S @@ -763,6 +763,10 @@ LnoR2Fix: #elif defined(__aarch64__) +#ifndef __has_feature +#define __has_feature(__feature) 0 +#endif + // // extern int __unw_getcontext(unw_context_t* thread_state) // diff --git a/llvm/docs/CommandGuide/llvm-ir2vec.rst b/llvm/docs/CommandGuide/llvm-ir2vec.rst index fc590a6..55fe75d 100644 --- a/llvm/docs/CommandGuide/llvm-ir2vec.rst +++ b/llvm/docs/CommandGuide/llvm-ir2vec.rst @@ -1,5 +1,5 @@ -llvm-ir2vec - IR2Vec Embedding Generation Tool -============================================== +llvm-ir2vec - IR2Vec and MIR2Vec Embedding Generation Tool +=========================================================== .. program:: llvm-ir2vec @@ -11,9 +11,9 @@ SYNOPSIS DESCRIPTION ----------- -:program:`llvm-ir2vec` is a standalone command-line tool for IR2Vec. It -generates IR2Vec embeddings for LLVM IR and supports triplet generation -for vocabulary training. +:program:`llvm-ir2vec` is a standalone command-line tool for IR2Vec and MIR2Vec. +It generates embeddings for both LLVM IR and Machine IR (MIR) and supports +triplet generation for vocabulary training. The tool provides three main subcommands: @@ -23,23 +23,33 @@ The tool provides three main subcommands: 2. **entities**: Generates entity mapping files (entity2id.txt) for vocabulary training. -3. **embeddings**: Generates IR2Vec embeddings using a trained vocabulary +3. **embeddings**: Generates IR2Vec or MIR2Vec embeddings using a trained vocabulary at different granularity levels (instruction, basic block, or function). +The tool supports two operation modes: + +* **LLVM IR mode** (``--mode=llvm``): Process LLVM IR bitcode files and generate + IR2Vec embeddings +* **Machine IR mode** (``--mode=mir``): Process Machine IR (.mir) files and generate + MIR2Vec embeddings + The tool is designed to facilitate machine learning applications that work with -LLVM IR by converting the IR into numerical representations that can be used by -ML models. The `triplets` subcommand generates numeric IDs directly instead of string -triplets, streamlining the training data preparation workflow. +LLVM IR or Machine IR by converting them into numerical representations that can +be used by ML models. The `triplets` subcommand generates numeric IDs directly +instead of string triplets, streamlining the training data preparation workflow. .. note:: - For information about using IR2Vec programmatically within LLVM passes and - the C++ API, see the `IR2Vec Embeddings <https://llvm.org/docs/MLGO.html#ir2vec-embeddings>`_ + For information about using IR2Vec and MIR2Vec programmatically within LLVM + passes and the C++ API, see the `IR2Vec Embeddings <https://llvm.org/docs/MLGO.html#ir2vec-embeddings>`_ section in the MLGO documentation. OPERATION MODES --------------- +The tool operates in two modes: **LLVM IR mode** and **Machine IR mode**. The mode +is selected using the ``--mode`` option (default: ``llvm``). + Triplet Generation and Entity Mapping Modes are used for preparing vocabulary and training data for knowledge graph embeddings. The Embedding Mode is used for generating embeddings from LLVM IR using a pre-trained vocabulary. @@ -89,18 +99,31 @@ Embedding Generation ~~~~~~~~~~~~~~~~~~~~ With the `embeddings` subcommand, :program:`llvm-ir2vec` uses a pre-trained vocabulary to -generate numerical embeddings for LLVM IR at different levels of granularity. +generate numerical embeddings for LLVM IR or Machine IR at different levels of granularity. + +Example Usage for LLVM IR: + +.. code-block:: bash + + llvm-ir2vec embeddings --mode=llvm --ir2vec-vocab-path=vocab.json --ir2vec-kind=symbolic --level=func input.bc -o embeddings.txt -Example Usage: +Example Usage for Machine IR: .. code-block:: bash - llvm-ir2vec embeddings --ir2vec-vocab-path=vocab.json --ir2vec-kind=symbolic --level=func input.bc -o embeddings.txt + llvm-ir2vec embeddings --mode=mir --mir2vec-vocab-path=vocab.json --level=func input.mir -o embeddings.txt OPTIONS ------- -Global options: +Common options (applicable to both LLVM IR and Machine IR modes): + +.. option:: --mode=<mode> + + Specify the operation mode. Valid values are: + + * ``llvm`` - Process LLVM IR bitcode files (default) + * ``mir`` - Process Machine IR (.mir) files .. option:: -o <filename> @@ -116,8 +139,8 @@ Subcommand-specific options: .. option:: <input-file> - The input LLVM IR or bitcode file to process. This positional argument is - required for the `embeddings` subcommand. + The input LLVM IR/bitcode file (.ll/.bc) or Machine IR file (.mir) to process. + This positional argument is required for the `embeddings` subcommand. .. option:: --level=<level> @@ -131,6 +154,8 @@ Subcommand-specific options: Process only the specified function instead of all functions in the module. +**IR2Vec-specific options** (for ``--mode=llvm``): + .. option:: --ir2vec-kind=<kind> Specify the kind of IR2Vec embeddings to generate. Valid values are: @@ -143,8 +168,8 @@ Subcommand-specific options: .. option:: --ir2vec-vocab-path=<path> - Specify the path to the vocabulary file (required for embedding generation). - The vocabulary file should be in JSON format and contain the trained + Specify the path to the IR2Vec vocabulary file (required for LLVM IR embedding + generation). The vocabulary file should be in JSON format and contain the trained vocabulary for embedding generation. See `llvm/lib/Analysis/models` for pre-trained vocabulary files. @@ -163,6 +188,35 @@ Subcommand-specific options: Specify the weight for argument embeddings (default: 0.2). This controls the relative importance of operand information in the final embedding. +**MIR2Vec-specific options** (for ``--mode=mir``): + +.. option:: --mir2vec-vocab-path=<path> + + Specify the path to the MIR2Vec vocabulary file (required for Machine IR + embedding generation). The vocabulary file should be in JSON format and + contain the trained vocabulary for embedding generation. + +.. option:: --mir2vec-kind=<kind> + + Specify the kind of MIR2Vec embeddings to generate. Valid values are: + + * ``symbolic`` - Generate symbolic embeddings (default) + +.. option:: --mir2vec-opc-weight=<weight> + + Specify the weight for machine opcode embeddings (default: 1.0). This controls + the relative importance of machine instruction opcodes in the final embedding. + +.. option:: --mir2vec-common-operand-weight=<weight> + + Specify the weight for common operand embeddings (default: 1.0). This controls + the relative importance of common operand types in the final embedding. + +.. option:: --mir2vec-reg-operand-weight=<weight> + + Specify the weight for register operand embeddings (default: 1.0). This controls + the relative importance of register operands in the final embedding. + **triplets** subcommand: @@ -240,3 +294,6 @@ SEE ALSO For more information about the IR2Vec algorithm and approach, see: `IR2Vec: LLVM IR Based Scalable Program Embeddings <https://doi.org/10.1145/3418463>`_. + +For more information about the MIR2Vec algorithm and approach, see: +`RL4ReAl: Reinforcement Learning for Register Allocation <https://doi.org/10.1145/3578360.3580273>`_. diff --git a/llvm/docs/Extensions.rst b/llvm/docs/Extensions.rst index 89a0e80..214323e 100644 --- a/llvm/docs/Extensions.rst +++ b/llvm/docs/Extensions.rst @@ -601,6 +601,35 @@ sees fit (generally the section that would provide the best locality). .. _CFI jump table: https://clang.llvm.org/docs/ControlFlowIntegrityDesign.html#forward-edge-cfi-for-indirect-function-calls +``SHT_LLVM_CALL_GRAPH`` Section (Call Graph) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This section is used to store the call graph. It has a type of +``SHT_LLVM_CALL_GRAPH`` (0x6fff4c0f). Details of call graph section layout +are described in :doc:`CallGraphSection`. + +For example: + +.. code-block:: gas + + .section ".llvm.callgraph","",@llvm_call_graph + .byte 0 + .byte 7 + .quad .Lball + .quad 0 + .byte 3 + .quad foo + .quad bar + .quad baz + .byte 3 + .quad 4524972987496481828 + .quad 3498816979441845844 + .quad 8646233951371320954 + +This indicates that ``ball`` calls ``foo``, ``bar`` and ``baz`` directly; +``ball`` indirectly calls functions whose types are ``4524972987496481828``, +``3498816979441845844`` and ``8646233951371320954``. + CodeView-Dependent ------------------ diff --git a/llvm/docs/MLGO.rst b/llvm/docs/MLGO.rst index bf3de11..2443835 100644 --- a/llvm/docs/MLGO.rst +++ b/llvm/docs/MLGO.rst @@ -434,8 +434,27 @@ The latter is also used in tests. There is no C++ implementation of a log reader. We do not have a scenario motivating one. -IR2Vec Embeddings -================= +Embeddings +========== + +LLVM provides embedding frameworks to generate vector representations of code +at different abstraction levels. These embeddings capture syntactic, semantic, +and structural properties of the code and can be used as features for machine +learning models in various compiler optimization tasks. + +Two embedding frameworks are available: + +- **IR2Vec**: Generates embeddings for LLVM IR +- **MIR2Vec**: Generates embeddings for Machine IR + +Both frameworks follow a similar architecture with vocabulary-based embedding +generation, where a vocabulary maps code entities to n-dimensional floating +point vectors. These embeddings can be computed at multiple granularity levels +(instruction, basic block, and function) and used for ML-guided compiler +optimizations. + +IR2Vec +------ IR2Vec is a program embedding approach designed specifically for LLVM IR. It is implemented as a function analysis pass in LLVM. The IR2Vec embeddings @@ -466,7 +485,7 @@ The core components are: compute embeddings for instructions, basic blocks, and functions. Using IR2Vec ------------- +^^^^^^^^^^^^ .. note:: @@ -526,7 +545,7 @@ embeddings can be computed and accessed via an ``ir2vec::Embedder`` instance. between different code snippets, or perform other analyses as needed. Further Details ---------------- +^^^^^^^^^^^^^^^ For more detailed information about the IR2Vec algorithm, its parameters, and advanced usage, please refer to the original paper: @@ -538,6 +557,123 @@ triplets from LLVM IR, see :doc:`CommandGuide/llvm-ir2vec`. The LLVM source code for ``IR2Vec`` can also be explored to understand the implementation details. +MIR2Vec +------- + +MIR2Vec is an extension of IR2Vec designed specifically for LLVM Machine IR +(MIR). It generates embeddings for machine-level instructions, basic blocks, +and functions. MIR2Vec operates on the target-specific machine representation, +capturing machine instruction semantics including opcodes, operands, and +register information at the machine level. + +MIR2Vec extends the vocabulary to include: + +- **Machine Opcodes**: Target-specific instruction opcodes derived from the + TargetInstrInfo, grouped by instruction semantics. + +- **Common Operands**: All common operand types (excluding register operands), + defined by the ``MachineOperand::MachineOperandType`` enum. + +- **Physical Register Classes**: Register classes defined by the target, + specialized for physical registers. + +- **Virtual Register Classes**: Register classes defined by the target, + specialized for virtual registers. + +The core components are: + +- **Vocabulary**: A mapping from machine IR entities (opcodes, operands, register + classes) to their vector representations. This is managed by + ``MIR2VecVocabLegacyAnalysis`` for the legacy pass manager, with a + ``MIR2VecVocabProvider`` that can be used standalone or wrapped by pass + managers. The vocabulary (.json file) contains sections for opcodes, common + operands, physical register classes, and virtual register classes. + + .. note:: + + The vocabulary file should contain these sections for it to be valid. + +- **Embedder**: A class (``mir2vec::MIREmbedder``) that uses the vocabulary to + compute embeddings for machine instructions, machine basic blocks, and + machine functions. Currently, ``SymbolicMIREmbedder`` is the available + implementation. + +Using MIR2Vec +^^^^^^^^^^^^^ + +.. note:: + + This section describes how to use MIR2Vec within LLVM passes. `llvm-ir2vec` + tool ` :doc:`CommandGuide/llvm-ir2vec` can be used for generating MIR2Vec + embeddings from Machine IR files (.mir), which can be useful for generating + embeddings outside of compiler passes. + +To generate MIR2Vec embeddings in a compiler pass, first obtain the vocabulary, +then create an embedder instance to compute and access embeddings. + +1. **Get the Vocabulary**: + In a MachineFunctionPass, get the vocabulary from the analysis: + + .. code-block:: c++ + + auto &VocabAnalysis = getAnalysis<MIR2VecVocabLegacyAnalysis>(); + auto VocabOrErr = VocabAnalysis.getMIR2VecVocabulary(*MF.getFunction().getParent()); + if (!VocabOrErr) { + // Handle error: vocabulary is not available or invalid + return; + } + const mir2vec::MIRVocabulary &Vocabulary = *VocabOrErr; + + Note that ``MIR2VecVocabLegacyAnalysis`` is an immutable pass. + +2. **Create Embedder instance**: + With the vocabulary, create an embedder for a specific machine function: + + .. code-block:: c++ + + // Assuming MF is a MachineFunction& + // For example, using MIR2VecKind::Symbolic: + std::unique_ptr<mir2vec::MIREmbedder> Emb = + mir2vec::MIREmbedder::create(MIR2VecKind::Symbolic, MF, Vocabulary); + + +3. **Compute and Access Embeddings**: + Call ``getMFunctionVector()`` to get the embedding for the machine function. + + .. code-block:: c++ + + mir2vec::Embedding FuncVector = Emb->getMFunctionVector(); + + Currently, ``MIREmbedder`` can generate embeddings at three levels: Machine + Instructions, Machine Basic Blocks, and Machine Functions. Appropriate + getters are provided to access the embeddings at these levels. + + .. note:: + + The validity of the ``MIREmbedder`` instance (and the embeddings it + generates) is tied to the machine function it is associated with. If the + machine function is modified, the embeddings may become stale and should + be recomputed accordingly. + +4. **Working with Embeddings:** + Embeddings are represented as ``std::vector<double>``. These vectors can be + used as features for machine learning models, compute similarity scores + between different code snippets, or perform other analyses as needed. + +Further Details +^^^^^^^^^^^^^^^ + +For more detailed information about the MIR2Vec algorithm, its parameters, and +advanced usage, please refer to the original paper: +`RL4ReAl: Reinforcement Learning for Register Allocation <https://doi.org/10.1145/3578360.3580273>`_. + +For information about using MIR2Vec tool for generating embeddings from +Machine IR, see :doc:`CommandGuide/llvm-ir2vec`. + +The LLVM source code for ``MIR2Vec`` can be explored to understand the +implementation details. See ``llvm/include/llvm/CodeGen/MIR2Vec.h`` and +``llvm/lib/CodeGen/MIR2Vec.cpp``. + Building with ML support ======================== diff --git a/llvm/include/llvm/ADT/IndexedMap.h b/llvm/include/llvm/ADT/IndexedMap.h index cda0316..638fe14 100644 --- a/llvm/include/llvm/ADT/IndexedMap.h +++ b/llvm/include/llvm/ADT/IndexedMap.h @@ -22,12 +22,21 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/identity.h" #include <cassert> namespace llvm { -template <typename T, typename ToIndexT = identity<unsigned>> class IndexedMap { +namespace detail { +template <class Ty> struct IdentityIndex { + using argument_type = Ty; + + Ty &operator()(Ty &self) const { return self; } + const Ty &operator()(const Ty &self) const { return self; } +}; +} // namespace detail + +template <typename T, typename ToIndexT = detail::IdentityIndex<unsigned>> +class IndexedMap { using IndexT = typename ToIndexT::argument_type; // Prefer SmallVector with zero inline storage over std::vector. IndexedMaps // can grow very large and SmallVector grows more efficiently as long as T diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h index efae6f3..ca0b918 100644 --- a/llvm/include/llvm/ADT/SmallVector.h +++ b/llvm/include/llvm/ADT/SmallVector.h @@ -734,6 +734,12 @@ public: void assign(const SmallVectorImpl &RHS) { assign(RHS.begin(), RHS.end()); } + template <typename U, + typename = std::enable_if_t<std::is_convertible_v<U, T>>> + void assign(ArrayRef<U> AR) { + assign(AR.begin(), AR.end()); + } + iterator erase(const_iterator CI) { // Just cast away constness because this is a non-const member function. iterator I = const_cast<iterator>(CI); @@ -1228,7 +1234,7 @@ public: } template <typename U, - typename = std::enable_if_t<std::is_convertible<U, T>::value>> + typename = std::enable_if_t<std::is_convertible_v<U, T>>> explicit SmallVector(ArrayRef<U> A) : SmallVectorImpl<T>(N) { this->append(A.begin(), A.end()); } diff --git a/llvm/include/llvm/ADT/identity.h b/llvm/include/llvm/ADT/identity.h deleted file mode 100644 index 88d033f..0000000 --- a/llvm/include/llvm/ADT/identity.h +++ /dev/null @@ -1,31 +0,0 @@ -//===- llvm/ADT/Identity.h - Provide std::identity from C++20 ---*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file provides an implementation of std::identity from C++20. -// -// No library is required when using these functions. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_ADT_IDENTITY_H -#define LLVM_ADT_IDENTITY_H - -namespace llvm { - -// Similar to `std::identity` from C++20. -template <class Ty> struct identity { - using is_transparent = void; - using argument_type = Ty; - - Ty &operator()(Ty &self) const { return self; } - const Ty &operator()(const Ty &self) const { return self; } -}; - -} // namespace llvm - -#endif // LLVM_ADT_IDENTITY_H diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index 8d0dc64..6ee6b666 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -1185,6 +1185,7 @@ enum : unsigned { SHT_LLVM_LTO = 0x6fff4c0c, // .llvm.lto for fat LTO. SHT_LLVM_JT_SIZES = 0x6fff4c0d, // LLVM jump tables sizes. SHT_LLVM_CFI_JUMP_TABLE = 0x6fff4c0e, // LLVM CFI jump table. + SHT_LLVM_CALL_GRAPH = 0x6fff4c0f, // LLVM Call Graph Section. // Android's experimental support for SHT_RELR sections. // https://android.googlesource.com/platform/bionic/+/b7feec74547f84559a1467aca02708ff61346d2a/libc/include/elf.h#512 SHT_ANDROID_RELR = 0x6fffff00, // Relocation entries; only offsets. diff --git a/llvm/include/llvm/CodeGen/MIR2Vec.h b/llvm/include/llvm/CodeGen/MIR2Vec.h index 4bcbad7..48bb0a5 100644 --- a/llvm/include/llvm/CodeGen/MIR2Vec.h +++ b/llvm/include/llvm/CodeGen/MIR2Vec.h @@ -7,9 +7,20 @@ //===----------------------------------------------------------------------===// /// /// \file -/// This file defines the MIR2Vec vocabulary -/// analysis(MIR2VecVocabLegacyAnalysis), the core mir2vec::MIREmbedder -/// interface for generating Machine IR embeddings, and related utilities. +/// This file defines the MIR2Vec framework for generating Machine IR +/// embeddings. +/// +/// Design Overview: +/// ---------------------- +/// 1. MIR2VecVocabProvider - Core vocabulary loading logic (no PM dependency) +/// - Can be used standalone or wrapped by the pass manager +/// - Requires MachineModuleInfo with parsed machine functions +/// +/// 2. MIR2VecVocabLegacyAnalysis - Pass manager wrapper (ImmutablePass) +/// - Integrated and used by llc -print-mir2vec +/// +/// 3. MIREmbedder - Generates embeddings from vocabulary +/// - SymbolicMIREmbedder: MIR2Vec embedding implementation /// /// MIR2Vec extends IR2Vec to support Machine IR embeddings. It represents the /// LLVM Machine IR as embeddings which can be used as input to machine learning @@ -306,26 +317,58 @@ public: } // namespace mir2vec +/// MIR2Vec vocabulary provider used by pass managers and standalone tools. +/// This class encapsulates the core vocabulary loading logic and can be used +/// independently of the pass manager infrastructure. For pass-based usage, +/// see MIR2VecVocabLegacyAnalysis. +/// +/// Note: This provider pattern makes new PM migration straightforward when +/// needed. A new PM analysis wrapper can be added that delegates to this +/// provider, similar to how MIR2VecVocabLegacyAnalysis currently wraps it. +class MIR2VecVocabProvider { + using VocabMap = std::map<std::string, mir2vec::Embedding>; + +public: + MIR2VecVocabProvider(const MachineModuleInfo &MMI) : MMI(MMI) {} + + Expected<mir2vec::MIRVocabulary> getVocabulary(const Module &M); + +private: + Error readVocabulary(VocabMap &OpcVocab, VocabMap &CommonOperandVocab, + VocabMap &PhyRegVocabMap, VocabMap &VirtRegVocabMap); + const MachineModuleInfo &MMI; +}; + /// Pass to analyze and populate MIR2Vec vocabulary from a module class MIR2VecVocabLegacyAnalysis : public ImmutablePass { using VocabVector = std::vector<mir2vec::Embedding>; using VocabMap = std::map<std::string, mir2vec::Embedding>; - std::optional<mir2vec::MIRVocabulary> Vocab; StringRef getPassName() const override; - Error readVocabulary(VocabMap &OpcVocab, VocabMap &CommonOperandVocab, - VocabMap &PhyRegVocabMap, VocabMap &VirtRegVocabMap); protected: void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired<MachineModuleInfoWrapperPass>(); AU.setPreservesAll(); } + std::unique_ptr<MIR2VecVocabProvider> Provider; public: static char ID; MIR2VecVocabLegacyAnalysis() : ImmutablePass(ID) {} - Expected<mir2vec::MIRVocabulary> getMIR2VecVocabulary(const Module &M); + + Expected<mir2vec::MIRVocabulary> getMIR2VecVocabulary(const Module &M) { + MachineModuleInfo &MMI = + getAnalysis<MachineModuleInfoWrapperPass>().getMMI(); + if (!Provider) + Provider = std::make_unique<MIR2VecVocabProvider>(MMI); + return Provider->getVocabulary(M); + } + + MIR2VecVocabProvider &getProvider() { + assert(Provider && "Provider not initialized"); + return *Provider; + } }; /// This pass prints the embeddings in the MIR2Vec vocabulary diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index 9c34bf1..b23093d 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -286,7 +286,6 @@ private: // FIXME: ObjCImageInfos and HeaderAddrs need to be cleared when // JITDylibs are removed. DenseMap<JITDylib *, ObjCImageInfo> ObjCImageInfos; - DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs; }; using GetJITDylibHeaderSendResultFn = diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 9e334d4..8e35109 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -3789,6 +3789,20 @@ def int_amdgcn_perm_pk16_b8_u4 : ClangBuiltin<"__builtin_amdgcn_perm_pk16_b8_u4" DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_i64_ty, llvm_i64_ty, llvm_v2i32_ty], [IntrNoMem, IntrSpeculatable]>; +class AMDGPUAddMinMax<LLVMType Ty, string Name> : ClangBuiltin<"__builtin_amdgcn_"#Name>, + DefaultAttrsIntrinsic<[Ty], [Ty, Ty, Ty, llvm_i1_ty /* clamp */], + [IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<3>>]>; + +def int_amdgcn_add_max_i32 : AMDGPUAddMinMax<llvm_i32_ty, "add_max_i32">; +def int_amdgcn_add_max_u32 : AMDGPUAddMinMax<llvm_i32_ty, "add_max_u32">; +def int_amdgcn_add_min_i32 : AMDGPUAddMinMax<llvm_i32_ty, "add_min_i32">; +def int_amdgcn_add_min_u32 : AMDGPUAddMinMax<llvm_i32_ty, "add_min_u32">; + +def int_amdgcn_pk_add_max_i16 : AMDGPUAddMinMax<llvm_v2i16_ty, "pk_add_max_i16">; +def int_amdgcn_pk_add_max_u16 : AMDGPUAddMinMax<llvm_v2i16_ty, "pk_add_max_u16">; +def int_amdgcn_pk_add_min_i16 : AMDGPUAddMinMax<llvm_v2i16_ty, "pk_add_min_i16">; +def int_amdgcn_pk_add_min_u16 : AMDGPUAddMinMax<llvm_v2i16_ty, "pk_add_min_u16">; + class AMDGPUCooperativeAtomicStore<LLVMType Ty> : Intrinsic < [], [llvm_anyptr_ty, // pointer to store to diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h index 98df06a..cdfee72 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -172,9 +172,11 @@ struct alignas(8) GlobalValueSummaryInfo { /// Add a summary corresponding to a global value definition in a module with /// the corresponding GUID. - void addSummary(std::unique_ptr<GlobalValueSummary> Summary) { - return SummaryList.push_back(std::move(Summary)); - } + inline void addSummary(std::unique_ptr<GlobalValueSummary> Summary); + + /// Verify that the HasLocal flag is consistent with the SummaryList. Should + /// only be called prior to index-based internalization and promotion. + inline void verifyLocal() const; private: /// List of global value summary structures for a particular value held @@ -183,6 +185,22 @@ private: /// compiling without sufficient distinguishing path, or (theoretically) hash /// collisions. Each summary is from a different module. GlobalValueSummaryList SummaryList; + + /// True if the SummaryList contains at least one summary with local linkage. + /// In most cases there should be only one, unless translation units with + /// same-named locals were compiled without distinguishing path. And generally + /// there should not be a mix of local and non-local summaries, because the + /// GUID for a local is computed with the path prepended and a ';' delimiter. + /// In extremely rare cases there could be a GUID hash collision. Having the + /// flag saves having to walk through all summaries to prove the existence or + /// not of any locals. + /// NOTE: this flag is set when the index is built. It does not reflect + /// index-based internalization and promotion decisions. Generally most + /// index-based analysis occurs before then, but any users should assert that + /// the withInternalizeAndPromote() flag is not set on the index. + /// TODO: Replace checks in various ThinLTO analyses that loop through all + /// summaries to handle the local case with a check of the flag. + bool HasLocal : 1; }; /// Map from global value GUID to corresponding summary structures. Use a @@ -219,6 +237,8 @@ struct ValueInfo { return getRef()->second.getSummaryList(); } + void verifyLocal() const { getRef()->second.verifyLocal(); } + // Even if the index is built with GVs available, we may not have one for // summary entries synthesized for profiled indirect call targets. bool hasName() const { return !haveGVs() || getValue(); } @@ -649,7 +669,23 @@ public: friend class ModuleSummaryIndex; }; -GlobalValueSummaryInfo::GlobalValueSummaryInfo(bool HaveGVs) : U(HaveGVs) {} +GlobalValueSummaryInfo::GlobalValueSummaryInfo(bool HaveGVs) + : U(HaveGVs), HasLocal(false) {} + +void GlobalValueSummaryInfo::addSummary( + std::unique_ptr<GlobalValueSummary> Summary) { + if (GlobalValue::isLocalLinkage(Summary->linkage())) + HasLocal = true; + return SummaryList.push_back(std::move(Summary)); +} + +void GlobalValueSummaryInfo::verifyLocal() const { + assert(HasLocal == + llvm::any_of(SummaryList, + [](const std::unique_ptr<GlobalValueSummary> &Summary) { + return GlobalValue::isLocalLinkage(Summary->linkage()); + })); +} /// Alias summary information. class AliasSummary : public GlobalValueSummary { diff --git a/llvm/include/llvm/Support/GlobPattern.h b/llvm/include/llvm/Support/GlobPattern.h index c1b4484..8cae6a3 100644 --- a/llvm/include/llvm/Support/GlobPattern.h +++ b/llvm/include/llvm/Support/GlobPattern.h @@ -63,21 +63,30 @@ public: // Returns true for glob pattern "*". Can be used to avoid expensive // preparation/acquisition of the input for match(). bool isTrivialMatchAll() const { - if (!Prefix.empty()) + if (PrefixSize) return false; - if (!Suffix.empty()) + if (SuffixSize) return false; if (SubGlobs.size() != 1) return false; return SubGlobs[0].getPat() == "*"; } - StringRef prefix() const { return Prefix; } - StringRef suffix() const { return Suffix; } + // The following functions are just shortcuts for faster matching. They are + // conservative to simplify implementations. + + // Returns plain prefix of the pattern. + StringRef prefix() const { return Pattern.take_front(PrefixSize); } + // Returns plain suffix of the pattern. + StringRef suffix() const { return Pattern.take_back(SuffixSize); } + // Returns the longest plain substring of the pattern between prefix and + // suffix. + StringRef longest_substr() const; private: - StringRef Prefix; - StringRef Suffix; + StringRef Pattern; + size_t PrefixSize = 0; + size_t SuffixSize = 0; struct SubGlobPattern { /// \param Pat the pattern to match against diff --git a/llvm/lib/Analysis/MemoryLocation.cpp b/llvm/lib/Analysis/MemoryLocation.cpp index 1c5f08e..edca387 100644 --- a/llvm/lib/Analysis/MemoryLocation.cpp +++ b/llvm/lib/Analysis/MemoryLocation.cpp @@ -288,6 +288,34 @@ MemoryLocation MemoryLocation::getForArgument(const CallBase *Call, LocationSize::precise(DL.getTypeStoreSize( II->getArgOperand(1)->getType())), AATags); + case Intrinsic::matrix_column_major_load: + case Intrinsic::matrix_column_major_store: { + bool IsLoad = II->getIntrinsicID() == Intrinsic::matrix_column_major_load; + assert(ArgIdx == (IsLoad ? 0 : 1) && "Invalid argument index"); + + auto *Stride = dyn_cast<ConstantInt>(II->getArgOperand(IsLoad ? 1 : 2)); + uint64_t Rows = + cast<ConstantInt>(II->getArgOperand(IsLoad ? 3 : 4))->getZExtValue(); + uint64_t Cols = + cast<ConstantInt>(II->getArgOperand(IsLoad ? 4 : 5))->getZExtValue(); + + // The stride is dynamic, so there's nothing we can say. + if (!Stride) + return MemoryLocation(Arg, LocationSize::afterPointer(), AATags); + + uint64_t ConstStride = Stride->getZExtValue(); + auto *VT = cast<VectorType>(IsLoad ? II->getType() + : II->getArgOperand(0)->getType()); + assert(Cols != 0 && "Matrix cannot have 0 columns"); + TypeSize Size = DL.getTypeAllocSize(VT->getScalarType()) * + (ConstStride * (Cols - 1) + Rows); + + // In the unstrided case, we have a precise size, ... + if (ConstStride == Rows) + return MemoryLocation(Arg, LocationSize::precise(Size), AATags); + // otherwise we merely obtain an upper bound. + return MemoryLocation(Arg, LocationSize::upperBound(Size), AATags); + } } assert( diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 1fe38d6..b49040b 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1862,15 +1862,19 @@ bool IRTranslator::translateVectorDeinterleave2Intrinsic( void IRTranslator::getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder) { + Value *Global = TLI->getSDagStackGuard(*MF->getFunction().getParent()); + if (!Global) { + LLVMContext &Ctx = MIRBuilder.getContext(); + Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard")); + MIRBuilder.buildUndef(DstReg); + return; + } + const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); MRI->setRegClass(DstReg, TRI->getPointerRegClass()); auto MIB = MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {}); - Value *Global = TLI->getSDagStackGuard(*MF->getFunction().getParent()); - if (!Global) - return; - unsigned AddrSpace = Global->getType()->getPointerAddressSpace(); LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace)); diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp index d6e8505..c3e0964 100644 --- a/llvm/lib/CodeGen/InlineSpiller.cpp +++ b/llvm/lib/CodeGen/InlineSpiller.cpp @@ -721,6 +721,9 @@ bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineInstr &MI) { // Allocate a new register for the remat. Register NewVReg = Edit->createFrom(Original); + // Constrain it to the register class of MI. + MRI.constrainRegClass(NewVReg, MRI.getRegClass(VirtReg.reg())); + // Finally we can rematerialize OrigMI before MI. SlotIndex DefIdx = Edit->rematerializeAt(*MI.getParent(), MI, NewVReg, RM, TRI); diff --git a/llvm/lib/CodeGen/MIR2Vec.cpp b/llvm/lib/CodeGen/MIR2Vec.cpp index 75ca06a..00b37e7 100644 --- a/llvm/lib/CodeGen/MIR2Vec.cpp +++ b/llvm/lib/CodeGen/MIR2Vec.cpp @@ -417,24 +417,39 @@ Expected<MIRVocabulary> MIRVocabulary::createDummyVocabForTest( } //===----------------------------------------------------------------------===// -// MIR2VecVocabLegacyAnalysis Implementation +// MIR2VecVocabProvider and MIR2VecVocabLegacyAnalysis //===----------------------------------------------------------------------===// -char MIR2VecVocabLegacyAnalysis::ID = 0; -INITIALIZE_PASS_BEGIN(MIR2VecVocabLegacyAnalysis, "mir2vec-vocab-analysis", - "MIR2Vec Vocabulary Analysis", false, true) -INITIALIZE_PASS_DEPENDENCY(MachineModuleInfoWrapperPass) -INITIALIZE_PASS_END(MIR2VecVocabLegacyAnalysis, "mir2vec-vocab-analysis", - "MIR2Vec Vocabulary Analysis", false, true) +Expected<mir2vec::MIRVocabulary> +MIR2VecVocabProvider::getVocabulary(const Module &M) { + VocabMap OpcVocab, CommonOperandVocab, PhyRegVocabMap, VirtRegVocabMap; -StringRef MIR2VecVocabLegacyAnalysis::getPassName() const { - return "MIR2Vec Vocabulary Analysis"; + if (Error Err = readVocabulary(OpcVocab, CommonOperandVocab, PhyRegVocabMap, + VirtRegVocabMap)) + return std::move(Err); + + for (const auto &F : M) { + if (F.isDeclaration()) + continue; + + if (auto *MF = MMI.getMachineFunction(F)) { + auto &Subtarget = MF->getSubtarget(); + if (const auto *TII = Subtarget.getInstrInfo()) + if (const auto *TRI = Subtarget.getRegisterInfo()) + return mir2vec::MIRVocabulary::create( + std::move(OpcVocab), std::move(CommonOperandVocab), + std::move(PhyRegVocabMap), std::move(VirtRegVocabMap), *TII, *TRI, + MF->getRegInfo()); + } + } + return createStringError(errc::invalid_argument, + "No machine functions found in module"); } -Error MIR2VecVocabLegacyAnalysis::readVocabulary(VocabMap &OpcodeVocab, - VocabMap &CommonOperandVocab, - VocabMap &PhyRegVocabMap, - VocabMap &VirtRegVocabMap) { +Error MIR2VecVocabProvider::readVocabulary(VocabMap &OpcodeVocab, + VocabMap &CommonOperandVocab, + VocabMap &PhyRegVocabMap, + VocabMap &VirtRegVocabMap) { if (VocabFile.empty()) return createStringError( errc::invalid_argument, @@ -483,49 +498,15 @@ Error MIR2VecVocabLegacyAnalysis::readVocabulary(VocabMap &OpcodeVocab, return Error::success(); } -Expected<mir2vec::MIRVocabulary> -MIR2VecVocabLegacyAnalysis::getMIR2VecVocabulary(const Module &M) { - if (Vocab.has_value()) - return std::move(Vocab.value()); - - VocabMap OpcMap, CommonOperandMap, PhyRegMap, VirtRegMap; - if (Error Err = - readVocabulary(OpcMap, CommonOperandMap, PhyRegMap, VirtRegMap)) - return std::move(Err); - - // Get machine module info to access machine functions and target info - MachineModuleInfo &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI(); - - // Find first available machine function to get target instruction info - for (const auto &F : M) { - if (F.isDeclaration()) - continue; - - if (auto *MF = MMI.getMachineFunction(F)) { - auto &Subtarget = MF->getSubtarget(); - const TargetInstrInfo *TII = Subtarget.getInstrInfo(); - if (!TII) { - return createStringError(errc::invalid_argument, - "No TargetInstrInfo available; cannot create " - "MIR2Vec vocabulary"); - } - - const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); - if (!TRI) { - return createStringError(errc::invalid_argument, - "No TargetRegisterInfo available; cannot " - "create MIR2Vec vocabulary"); - } - - return mir2vec::MIRVocabulary::create( - std::move(OpcMap), std::move(CommonOperandMap), std::move(PhyRegMap), - std::move(VirtRegMap), *TII, *TRI, MF->getRegInfo()); - } - } +char MIR2VecVocabLegacyAnalysis::ID = 0; +INITIALIZE_PASS_BEGIN(MIR2VecVocabLegacyAnalysis, "mir2vec-vocab-analysis", + "MIR2Vec Vocabulary Analysis", false, true) +INITIALIZE_PASS_DEPENDENCY(MachineModuleInfoWrapperPass) +INITIALIZE_PASS_END(MIR2VecVocabLegacyAnalysis, "mir2vec-vocab-analysis", + "MIR2Vec Vocabulary Analysis", false, true) - // No machine functions available - return error - return createStringError(errc::invalid_argument, - "No machine functions found in module"); +StringRef MIR2VecVocabLegacyAnalysis::getPassName() const { + return "MIR2Vec Vocabulary Analysis"; } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index dcf2df3..d57c5fb 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3131,12 +3131,16 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, if (TLI.useLoadStackGuardNode(M)) { Guard = getLoadStackGuard(DAG, dl, Chain); } else { - const Value *IRGuard = TLI.getSDagStackGuard(M); - SDValue GuardPtr = getValue(IRGuard); - - Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr, - MachinePointerInfo(IRGuard, 0), Align, - MachineMemOperand::MOVolatile); + if (const Value *IRGuard = TLI.getSDagStackGuard(M)) { + SDValue GuardPtr = getValue(IRGuard); + Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr, + MachinePointerInfo(IRGuard, 0), Align, + MachineMemOperand::MOVolatile); + } else { + LLVMContext &Ctx = *DAG.getContext(); + Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard")); + Guard = DAG.getPOISON(PtrMemTy); + } } // Perform the comparison via a getsetcc. @@ -7324,6 +7328,13 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy); } else { const Value *Global = TLI.getSDagStackGuard(M); + if (!Global) { + LLVMContext &Ctx = *DAG.getContext(); + Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard")); + setValue(&I, DAG.getPOISON(PtrTy)); + return; + } + Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType()); Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global), MachinePointerInfo(Global, 0), Align, diff --git a/llvm/lib/Frontend/HLSL/CBuffer.cpp b/llvm/lib/Frontend/HLSL/CBuffer.cpp index 407b6ad..1f53c87 100644 --- a/llvm/lib/Frontend/HLSL/CBuffer.cpp +++ b/llvm/lib/Frontend/HLSL/CBuffer.cpp @@ -43,8 +43,13 @@ std::optional<CBufferMetadata> CBufferMetadata::get(Module &M) { for (const MDNode *MD : CBufMD->operands()) { assert(MD->getNumOperands() && "Invalid cbuffer metadata"); - auto *Handle = cast<GlobalVariable>( - cast<ValueAsMetadata>(MD->getOperand(0))->getValue()); + // For an unused cbuffer, the handle may have been optimized out + Metadata *OpMD = MD->getOperand(0); + if (!OpMD) + continue; + + auto *Handle = + cast<GlobalVariable>(cast<ValueAsMetadata>(OpMD)->getValue()); CBufferMapping &Mapping = Result->Mappings.emplace_back(Handle); for (int I = 1, E = MD->getNumOperands(); I < E; ++I) { diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 72ae064..86780e1 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -477,6 +477,10 @@ static void thinLTOInternalizeAndPromoteGUID( return !GlobalValue::isLocalLinkage(Summary->linkage()); }); + // Before performing index-based internalization and promotion for this GUID, + // the local flag should be consistent with the summary list linkage types. + VI.verifyLocal(); + for (auto &S : VI.getSummaryList()) { // First see if we need to promote an internal value because it is not // exported. diff --git a/llvm/lib/MC/MCAsmInfoELF.cpp b/llvm/lib/MC/MCAsmInfoELF.cpp index 98090d3..6670971 100644 --- a/llvm/lib/MC/MCAsmInfoELF.cpp +++ b/llvm/lib/MC/MCAsmInfoELF.cpp @@ -197,6 +197,8 @@ void MCAsmInfoELF::printSwitchToSection(const MCSection &Section, OS << "llvm_jt_sizes"; else if (Sec.Type == ELF::SHT_LLVM_CFI_JUMP_TABLE) OS << "llvm_cfi_jump_table"; + else if (Sec.Type == ELF::SHT_LLVM_CALL_GRAPH) + OS << "llvm_call_graph"; else OS << "0x" << Twine::utohexstr(Sec.Type); diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp index aee3c3b..b2f5000 100644 --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -554,7 +554,7 @@ void MCObjectFileInfo::initELFMCObjectFileInfo(const Triple &T, bool Large) { Ctx->getELFSection(".sframe", ELF::SHT_GNU_SFRAME, ELF::SHF_ALLOC); CallGraphSection = - Ctx->getELFSection(".llvm.callgraph", ELF::SHT_PROGBITS, 0); + Ctx->getELFSection(".llvm.callgraph", ELF::SHT_LLVM_CALL_GRAPH, 0); StackSizesSection = Ctx->getELFSection(".stack_sizes", ELF::SHT_PROGBITS, 0); @@ -1172,7 +1172,7 @@ MCObjectFileInfo::getCallGraphSection(const MCSection &TextSec) const { } return Ctx->getELFSection( - ".llvm.callgraph", ELF::SHT_PROGBITS, Flags, 0, GroupName, + ".llvm.callgraph", ELF::SHT_LLVM_CALL_GRAPH, Flags, 0, GroupName, /*IsComdat=*/true, ElfSec.getUniqueID(), static_cast<const MCSymbolELF *>(TextSec.getBeginSymbol())); } diff --git a/llvm/lib/MC/MCParser/ELFAsmParser.cpp b/llvm/lib/MC/MCParser/ELFAsmParser.cpp index 6195355..1a3752f 100644 --- a/llvm/lib/MC/MCParser/ELFAsmParser.cpp +++ b/llvm/lib/MC/MCParser/ELFAsmParser.cpp @@ -637,6 +637,8 @@ EndStmt: Type = ELF::SHT_LLVM_JT_SIZES; else if (TypeName == "llvm_cfi_jump_table") Type = ELF::SHT_LLVM_CFI_JUMP_TABLE; + else if (TypeName == "llvm_call_graph") + Type = ELF::SHT_LLVM_CALL_GRAPH; else if (TypeName.getAsInteger(0, Type)) return TokError("unknown section type"); } diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp index f256e7b..6da97f9 100644 --- a/llvm/lib/Object/ELF.cpp +++ b/llvm/lib/Object/ELF.cpp @@ -322,6 +322,7 @@ StringRef llvm::object::getELFSectionTypeName(uint32_t Machine, unsigned Type) { STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_LTO); STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_JT_SIZES) STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_CFI_JUMP_TABLE) + STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_CALL_GRAPH); STRINGIFY_ENUM_CASE(ELF, SHT_GNU_SFRAME); STRINGIFY_ENUM_CASE(ELF, SHT_GNU_ATTRIBUTES); STRINGIFY_ENUM_CASE(ELF, SHT_GNU_HASH); diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp index c3a27c9..f8a84b0 100644 --- a/llvm/lib/ObjectYAML/ELFYAML.cpp +++ b/llvm/lib/ObjectYAML/ELFYAML.cpp @@ -744,6 +744,7 @@ void ScalarEnumerationTraits<ELFYAML::ELF_SHT>::enumeration( ECase(SHT_LLVM_BB_ADDR_MAP); ECase(SHT_LLVM_OFFLOADING); ECase(SHT_LLVM_LTO); + ECase(SHT_LLVM_CALL_GRAPH); ECase(SHT_GNU_SFRAME); ECase(SHT_GNU_ATTRIBUTES); ECase(SHT_GNU_HASH); diff --git a/llvm/lib/Support/GlobPattern.cpp b/llvm/lib/Support/GlobPattern.cpp index 0ecf47d..2715229 100644 --- a/llvm/lib/Support/GlobPattern.cpp +++ b/llvm/lib/Support/GlobPattern.cpp @@ -132,24 +132,70 @@ parseBraceExpansions(StringRef S, std::optional<size_t> MaxSubPatterns) { return std::move(SubPatterns); } +static StringRef maxPlainSubstring(StringRef S) { + StringRef Best; + while (!S.empty()) { + size_t PrefixSize = S.find_first_of("?*[{\\"); + if (PrefixSize == std::string::npos) + PrefixSize = S.size(); + + if (Best.size() < PrefixSize) + Best = S.take_front(PrefixSize); + + S = S.drop_front(PrefixSize); + + // It's impossible, as the first and last characters of the input string + // must be Glob special characters, otherwise they would be parts of + // the prefix or the suffix. + assert(!S.empty()); + + switch (S.front()) { + case '\\': + S = S.drop_front(2); + break; + case '[': { + // Drop '[' and the first character which can be ']'. + S = S.drop_front(2); + size_t EndBracket = S.find_first_of("]"); + // Should not be possible, SubGlobPattern::create should fail on invalid + // pattern before we get here. + assert(EndBracket != std::string::npos); + S = S.drop_front(EndBracket + 1); + break; + } + case '{': + // TODO: implement. + // Fallback to whatever is best for now. + return Best; + default: + S = S.drop_front(1); + } + } + + return Best; +} + Expected<GlobPattern> GlobPattern::create(StringRef S, std::optional<size_t> MaxSubPatterns) { GlobPattern Pat; + Pat.Pattern = S; // Store the prefix that does not contain any metacharacter. - size_t PrefixSize = S.find_first_of("?*[{\\"); - Pat.Prefix = S.substr(0, PrefixSize); - if (PrefixSize == std::string::npos) + Pat.PrefixSize = S.find_first_of("?*[{\\"); + if (Pat.PrefixSize == std::string::npos) { + Pat.PrefixSize = S.size(); return Pat; - S = S.substr(PrefixSize); + } + S = S.substr(Pat.PrefixSize); // Just in case we stop on unmatched opening brackets. size_t SuffixStart = S.find_last_of("?*[]{}\\"); assert(SuffixStart != std::string::npos); if (S[SuffixStart] == '\\') ++SuffixStart; - ++SuffixStart; - Pat.Suffix = S.substr(SuffixStart); + if (SuffixStart < S.size()) + ++SuffixStart; + Pat.SuffixSize = S.size() - SuffixStart; S = S.substr(0, SuffixStart); SmallVector<std::string, 1> SubPats; @@ -199,10 +245,15 @@ GlobPattern::SubGlobPattern::create(StringRef S) { return Pat; } +StringRef GlobPattern::longest_substr() const { + return maxPlainSubstring( + Pattern.drop_front(PrefixSize).drop_back(SuffixSize)); +} + bool GlobPattern::match(StringRef S) const { - if (!S.consume_front(Prefix)) + if (!S.consume_front(prefix())) return false; - if (!S.consume_back(Suffix)) + if (!S.consume_back(suffix())) return false; if (SubGlobs.empty() && S.empty()) return true; diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index ea32748..1c8383c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -1430,6 +1430,18 @@ def FeatureAddSubU64Insts def FeatureMadU32Inst : SubtargetFeature<"mad-u32-inst", "HasMadU32Inst", "true", "Has v_mad_u32 instruction">; +def FeatureAddMinMaxInsts : SubtargetFeature<"add-min-max-insts", + "HasAddMinMaxInsts", + "true", + "Has v_add_{min|max}_{i|u}32 instructions" +>; + +def FeaturePkAddMinMaxInsts : SubtargetFeature<"pk-add-min-max-insts", + "HasPkAddMinMaxInsts", + "true", + "Has v_pk_add_{min|max}_{i|u}16 instructions" +>; + def FeatureMemToLDSLoad : SubtargetFeature<"vmem-to-lds-load-insts", "HasVMemToLDSLoad", "true", @@ -2115,6 +2127,8 @@ def FeatureISAVersion12_50 : FeatureSet< FeatureLshlAddU64Inst, FeatureAddSubU64Insts, FeatureMadU32Inst, + FeatureAddMinMaxInsts, + FeaturePkAddMinMaxInsts, FeatureLdsBarrierArriveAtomic, FeatureSetPrioIncWgInst, Feature45BitNumRecordsBufferResource, @@ -2658,11 +2672,11 @@ def HasFmaakFmamkF64Insts : def HasAddMinMaxInsts : Predicate<"Subtarget->hasAddMinMaxInsts()">, - AssemblerPredicate<(any_of FeatureGFX1250Insts)>; + AssemblerPredicate<(any_of FeatureAddMinMaxInsts)>; def HasPkAddMinMaxInsts : Predicate<"Subtarget->hasPkAddMinMaxInsts()">, - AssemblerPredicate<(any_of FeatureGFX1250Insts)>; + AssemblerPredicate<(any_of FeaturePkAddMinMaxInsts)>; def HasPkMinMax3Insts : Predicate<"Subtarget->hasPkMinMax3Insts()">, diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp index 56807a4..54ba2f8 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp @@ -4835,6 +4835,14 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case Intrinsic::amdgcn_perm_pk16_b4_u4: case Intrinsic::amdgcn_perm_pk16_b6_u4: case Intrinsic::amdgcn_perm_pk16_b8_u4: + case Intrinsic::amdgcn_add_max_i32: + case Intrinsic::amdgcn_add_max_u32: + case Intrinsic::amdgcn_add_min_i32: + case Intrinsic::amdgcn_add_min_u32: + case Intrinsic::amdgcn_pk_add_max_i16: + case Intrinsic::amdgcn_pk_add_max_u16: + case Intrinsic::amdgcn_pk_add_min_i16: + case Intrinsic::amdgcn_pk_add_min_u16: return getDefaultMappingVOP(MI); case Intrinsic::amdgcn_log: case Intrinsic::amdgcn_exp2: diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index a466780..ac660d5 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -277,6 +277,8 @@ protected: bool HasLshlAddU64Inst = false; bool HasAddSubU64Insts = false; bool HasMadU32Inst = false; + bool HasAddMinMaxInsts = false; + bool HasPkAddMinMaxInsts = false; bool HasPointSampleAccel = false; bool HasLdsBarrierArriveAtomic = false; bool HasSetPrioIncWgInst = false; @@ -1567,10 +1569,10 @@ public: bool hasIntMinMax64() const { return GFX1250Insts; } // \returns true if the target has V_ADD_{MIN|MAX}_{I|U}32 instructions. - bool hasAddMinMaxInsts() const { return GFX1250Insts; } + bool hasAddMinMaxInsts() const { return HasAddMinMaxInsts; } // \returns true if the target has V_PK_ADD_{MIN|MAX}_{I|U}16 instructions. - bool hasPkAddMinMaxInsts() const { return GFX1250Insts; } + bool hasPkAddMinMaxInsts() const { return HasPkAddMinMaxInsts; } // \returns true if the target has V_PK_{MIN|MAX}3_{I|U}16 instructions. bool hasPkMinMax3Insts() const { return GFX1250Insts; } diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td index 7cce033..ee10190 100644 --- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td @@ -775,10 +775,10 @@ let SubtargetPredicate = HasMinimum3Maximum3F16, ReadsModeReg = 0 in { } // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 let SubtargetPredicate = HasAddMinMaxInsts, isCommutable = 1, isReMaterializable = 1 in { - defm V_ADD_MAX_I32 : VOP3Inst <"v_add_max_i32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>; - defm V_ADD_MAX_U32 : VOP3Inst <"v_add_max_u32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>; - defm V_ADD_MIN_I32 : VOP3Inst <"v_add_min_i32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>; - defm V_ADD_MIN_U32 : VOP3Inst <"v_add_min_u32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>; + defm V_ADD_MAX_I32 : VOP3Inst <"v_add_max_i32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>, int_amdgcn_add_max_i32>; + defm V_ADD_MAX_U32 : VOP3Inst <"v_add_max_u32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>, int_amdgcn_add_max_u32>; + defm V_ADD_MIN_I32 : VOP3Inst <"v_add_min_i32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>, int_amdgcn_add_min_i32>; + defm V_ADD_MIN_U32 : VOP3Inst <"v_add_min_u32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>, int_amdgcn_add_min_u32>; } defm V_ADD_I16 : VOP3Inst_t16 <"v_add_i16", VOP_I16_I16_I16>; diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td index 6500fce..c4692b7 100644 --- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td @@ -75,7 +75,7 @@ multiclass VOP3PInst<string OpName, VOPProfile P, SDPatternOperator node = null_frag, bit IsDOT = 0> { def NAME : VOP3P_Pseudo<OpName, P, !if (P.HasModifiers, - getVOP3PModPat<P, node, IsDOT, IsDOT>.ret, + getVOP3PModPat<P, node, !or(P.EnableClamp, IsDOT), IsDOT>.ret, getVOP3Pat<P, node>.ret)>; let SubtargetPredicate = isGFX11Plus in { if P.HasExtVOP3DPP then @@ -434,15 +434,16 @@ defm : MadFmaMixFP16Pats_t16<fma, V_FMA_MIX_BF16_t16>; } // End SubtargetPredicate = HasFmaMixBF16Insts def PK_ADD_MINMAX_Profile : VOP3P_Profile<VOP_V2I16_V2I16_V2I16_V2I16, VOP3_PACKED> { - let HasModifiers = 0; + let HasNeg = 0; + let EnableClamp = 1; } let isCommutable = 1, isReMaterializable = 1 in { let SubtargetPredicate = HasPkAddMinMaxInsts in { -defm V_PK_ADD_MAX_I16 : VOP3PInst<"v_pk_add_max_i16", PK_ADD_MINMAX_Profile>; -defm V_PK_ADD_MAX_U16 : VOP3PInst<"v_pk_add_max_u16", PK_ADD_MINMAX_Profile>; -defm V_PK_ADD_MIN_I16 : VOP3PInst<"v_pk_add_min_i16", PK_ADD_MINMAX_Profile>; -defm V_PK_ADD_MIN_U16 : VOP3PInst<"v_pk_add_min_u16", PK_ADD_MINMAX_Profile>; +defm V_PK_ADD_MAX_I16 : VOP3PInst<"v_pk_add_max_i16", PK_ADD_MINMAX_Profile, int_amdgcn_pk_add_max_i16>; +defm V_PK_ADD_MAX_U16 : VOP3PInst<"v_pk_add_max_u16", PK_ADD_MINMAX_Profile, int_amdgcn_pk_add_max_u16>; +defm V_PK_ADD_MIN_I16 : VOP3PInst<"v_pk_add_min_i16", PK_ADD_MINMAX_Profile, int_amdgcn_pk_add_min_i16>; +defm V_PK_ADD_MIN_U16 : VOP3PInst<"v_pk_add_min_u16", PK_ADD_MINMAX_Profile, int_amdgcn_pk_add_min_u16>; } let SubtargetPredicate = HasPkMinMax3Insts in { defm V_PK_MAX3_I16 : VOP3PInst<"v_pk_max3_i16", PK_ADD_MINMAX_Profile>; diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index f7deeaf..ca4a655 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -2614,6 +2614,9 @@ static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT, if ((Result = lowerVECTOR_SHUFFLE_XVSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget))) return Result; + // Try to widen vectors to gain more optimization opportunities. + if (SDValue NewShuffle = widenShuffleMask(DL, Mask, VT, V1, V2, DAG)) + return NewShuffle; if ((Result = lowerVECTOR_SHUFFLE_XVPERMI(DL, Mask, VT, V1, DAG, Subtarget))) return Result; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td index f7d1a09..b9c5b75 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td @@ -668,4 +668,38 @@ foreach vti = NoGroupBF16Vectors in { def : Pat<(vti.Scalar (extractelt (vti.Vector vti.RegClass:$rs2), 0)), (vfmv_f_s_inst vti.RegClass:$rs2, vti.Log2SEW)>; } + +let Predicates = [HasStdExtZvfbfa] in { + foreach fvtiToFWti = AllWidenableBF16ToFloatVectors in { + defvar fvti = fvtiToFWti.Vti; + defvar fwti = fvtiToFWti.Wti; + def : Pat<(fwti.Vector (any_riscv_fpextend_vl + (fvti.Vector fvti.RegClass:$rs1), + (fvti.Mask VMV0:$vm), + VLOpFrag)), + (!cast<Instruction>("PseudoVFWCVT_F_F_ALT_V_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") + (fwti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs1, + (fvti.Mask VMV0:$vm), + GPR:$vl, fvti.Log2SEW, TA_MA)>; + + def : Pat<(fvti.Vector (any_riscv_fpround_vl + (fwti.Vector fwti.RegClass:$rs1), + (fwti.Mask VMV0:$vm), VLOpFrag)), + (!cast<Instruction>("PseudoVFNCVT_F_F_ALT_W_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") + (fvti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1, + (fwti.Mask VMV0:$vm), + // Value to indicate no rounding mode change in + // RISCVInsertReadWriteCSR + FRM_DYN, + GPR:$vl, fvti.Log2SEW, TA_MA)>; + def : Pat<(fvti.Vector (fpround (fwti.Vector fwti.RegClass:$rs1))), + (!cast<Instruction>("PseudoVFNCVT_F_F_ALT_W_"#fvti.LMul.MX#"_E"#fvti.SEW) + (fvti.Vector (IMPLICIT_DEF)), + fwti.RegClass:$rs1, + // Value to indicate no rounding mode change in + // RISCVInsertReadWriteCSR + FRM_DYN, + fvti.AVL, fvti.Log2SEW, TA_MA)>; + } +} } // Predicates = [HasStdExtZvfbfa] diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp index 62a3c88..975a271 100644 --- a/llvm/lib/TargetParser/TargetParser.cpp +++ b/llvm/lib/TargetParser/TargetParser.cpp @@ -433,6 +433,8 @@ static void fillAMDGCNFeatureMap(StringRef GPU, const Triple &T, Features["fp8e5m3-insts"] = true; Features["permlane16-swap"] = true; Features["ashr-pk-insts"] = true; + Features["add-min-max-insts"] = true; + Features["pk-add-min-max-insts"] = true; Features["atomic-buffer-pk-add-bf16-inst"] = true; Features["vmem-pref-insts"] = true; Features["atomic-fadd-rtn-insts"] = true; diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index d2c100c9..3356516 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7231,6 +7231,9 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan( return DenseMap<const SCEV *, Value *>(); } + VPlanTransforms::narrowInterleaveGroups( + BestVPlan, BestVF, + TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)); VPlanTransforms::removeDeadRecipes(BestVPlan); VPlanTransforms::convertToConcreteRecipes(BestVPlan); @@ -8199,10 +8202,6 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, if (CM.foldTailWithEVL()) VPlanTransforms::runPass(VPlanTransforms::addExplicitVectorLength, *Plan, CM.getMaxSafeElements()); - - if (auto P = VPlanTransforms::narrowInterleaveGroups(*Plan, TTI)) - VPlans.push_back(std::move(P)); - assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid"); VPlans.push_back(std::move(Plan)); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index c95c887..428a8f4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1191,7 +1191,6 @@ VPlan *VPlan::duplicate() { } Old2NewVPValues[&VectorTripCount] = &NewPlan->VectorTripCount; Old2NewVPValues[&VF] = &NewPlan->VF; - Old2NewVPValues[&UF] = &NewPlan->UF; Old2NewVPValues[&VFxUF] = &NewPlan->VFxUF; if (BackedgeTakenCount) { NewPlan->BackedgeTakenCount = new VPValue(); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 167ba55..06bea2f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -4152,9 +4152,6 @@ class VPlan { /// Represents the vectorization factor of the loop. VPValue VF; - /// Represents the symbolic unroll factor of the loop. - VPValue UF; - /// Represents the loop-invariant VF * UF of the vector loop region. VPValue VFxUF; @@ -4308,9 +4305,6 @@ public: VPValue &getVF() { return VF; }; const VPValue &getVF() const { return VF; }; - /// Returns the symbolic UF of the vector loop region. - VPValue &getSymbolicUF() { return UF; }; - /// Returns VF * UF of the vector loop region. VPValue &getVFxUF() { return VFxUF; } @@ -4320,12 +4314,6 @@ public: void addVF(ElementCount VF) { VFs.insert(VF); } - /// Remove \p VF from the plan. - void removeVF(ElementCount VF) { - assert(hasVF(VF) && "tried to remove VF not present in plan"); - VFs.remove(VF); - } - void setVF(ElementCount VF) { assert(hasVF(VF) && "Cannot set VF not already in plan"); VFs.clear(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 48cf763..f5a3af4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3956,9 +3956,6 @@ void VPlanTransforms::materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH, // used. // TODO: Assert that they aren't used. - VPValue *UF = Plan.getOrAddLiveIn(ConstantInt::get(TCTy, Plan.getUF())); - Plan.getSymbolicUF().replaceAllUsesWith(UF); - // If there are no users of the runtime VF, compute VFxUF by constant folding // the multiplication of VF and UF. if (VF.getNumUsers() == 0) { @@ -3978,6 +3975,7 @@ void VPlanTransforms::materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH, } VF.replaceAllUsesWith(RuntimeVF); + VPValue *UF = Plan.getOrAddLiveIn(ConstantInt::get(TCTy, Plan.getUF())); VPValue *MulByUF = Builder.createNaryOp(Instruction::Mul, {RuntimeVF, UF}); VFxUF.replaceAllUsesWith(MulByUF); } @@ -4045,14 +4043,14 @@ static bool canNarrowLoad(VPWidenRecipe *WideMember0, unsigned OpIdx, return false; } -/// Returns VF from \p VFs if \p IR is a full interleave group with factor and -/// number of members both equal to VF. The interleave group must also access -/// the full vector width. -static std::optional<ElementCount> isConsecutiveInterleaveGroup( - VPInterleaveRecipe *InterleaveR, ArrayRef<ElementCount> VFs, - VPTypeAnalysis &TypeInfo, const TargetTransformInfo &TTI) { +/// Returns true if \p IR is a full interleave group with factor and number of +/// members both equal to \p VF. The interleave group must also access the full +/// vector width \p VectorRegWidth. +static bool isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR, + unsigned VF, VPTypeAnalysis &TypeInfo, + unsigned VectorRegWidth) { if (!InterleaveR || InterleaveR->getMask()) - return std::nullopt; + return false; Type *GroupElementTy = nullptr; if (InterleaveR->getStoredValues().empty()) { @@ -4061,7 +4059,7 @@ static std::optional<ElementCount> isConsecutiveInterleaveGroup( [&TypeInfo, GroupElementTy](VPValue *Op) { return TypeInfo.inferScalarType(Op) == GroupElementTy; })) - return std::nullopt; + return false; } else { GroupElementTy = TypeInfo.inferScalarType(InterleaveR->getStoredValues()[0]); @@ -4069,27 +4067,13 @@ static std::optional<ElementCount> isConsecutiveInterleaveGroup( [&TypeInfo, GroupElementTy](VPValue *Op) { return TypeInfo.inferScalarType(Op) == GroupElementTy; })) - return std::nullopt; + return false; } - auto GetVectorWidthForVF = [&TTI](ElementCount VF) { - TypeSize Size = TTI.getRegisterBitWidth( - VF.isFixed() ? TargetTransformInfo::RGK_FixedWidthVector - : TargetTransformInfo::RGK_ScalableVector); - assert(Size.isScalable() == VF.isScalable() && - "if Size is scalable, VF must to and vice versa"); - return Size.getKnownMinValue(); - }; - - for (ElementCount VF : VFs) { - unsigned MinVal = VF.getKnownMinValue(); - unsigned GroupSize = GroupElementTy->getScalarSizeInBits() * MinVal; - auto IG = InterleaveR->getInterleaveGroup(); - if (IG->getFactor() == MinVal && IG->getNumMembers() == MinVal && - GroupSize == GetVectorWidthForVF(VF)) - return {VF}; - } - return std::nullopt; + unsigned GroupSize = GroupElementTy->getScalarSizeInBits() * VF; + auto IG = InterleaveR->getInterleaveGroup(); + return IG->getFactor() == VF && IG->getNumMembers() == VF && + GroupSize == VectorRegWidth; } /// Returns true if \p VPValue is a narrow VPValue. @@ -4100,18 +4084,16 @@ static bool isAlreadyNarrow(VPValue *VPV) { return RepR && RepR->isSingleScalar(); } -std::unique_ptr<VPlan> -VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, - const TargetTransformInfo &TTI) { - using namespace llvm::VPlanPatternMatch; +void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF, + unsigned VectorRegWidth) { VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion(); - if (!VectorLoop) - return nullptr; + return; VPTypeAnalysis TypeInfo(Plan); + + unsigned VFMinVal = VF.getKnownMinValue(); SmallVector<VPInterleaveRecipe *> StoreGroups; - std::optional<ElementCount> VFToOptimize; for (auto &R : *VectorLoop->getEntryBasicBlock()) { if (isa<VPCanonicalIVPHIRecipe>(&R) || match(&R, m_BranchOnCount())) continue; @@ -4125,33 +4107,30 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, // * recipes writing to memory except interleave groups // Only support plans with a canonical induction phi. if (R.isPhi()) - return nullptr; + return; auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(&R); if (R.mayWriteToMemory() && !InterleaveR) - return nullptr; + return; + + // Do not narrow interleave groups if there are VectorPointer recipes and + // the plan was unrolled. The recipe implicitly uses VF from + // VPTransformState. + // TODO: Remove restriction once the VF for the VectorPointer offset is + // modeled explicitly as operand. + if (isa<VPVectorPointerRecipe>(&R) && Plan.getUF() > 1) + return; // All other ops are allowed, but we reject uses that cannot be converted // when checking all allowed consumers (store interleave groups) below. if (!InterleaveR) continue; - // Try to find a single VF, where all interleave groups are consecutive and - // saturate the full vector width. If we already have a candidate VF, check - // if it is applicable for the current InterleaveR, otherwise look for a - // suitable VF across the Plans VFs. - // - if (VFToOptimize) { - if (!isConsecutiveInterleaveGroup(InterleaveR, {*VFToOptimize}, TypeInfo, - TTI)) - return nullptr; - } else { - if (auto VF = isConsecutiveInterleaveGroup( - InterleaveR, to_vector(Plan.vectorFactors()), TypeInfo, TTI)) - VFToOptimize = *VF; - else - return nullptr; - } + // Bail out on non-consecutive interleave groups. + if (!isConsecutiveInterleaveGroup(InterleaveR, VFMinVal, TypeInfo, + VectorRegWidth)) + return; + // Skip read interleave groups. if (InterleaveR->getStoredValues().empty()) continue; @@ -4185,34 +4164,24 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, auto *WideMember0 = dyn_cast_or_null<VPWidenRecipe>( InterleaveR->getStoredValues()[0]->getDefiningRecipe()); if (!WideMember0) - return nullptr; + return; for (const auto &[I, V] : enumerate(InterleaveR->getStoredValues())) { auto *R = dyn_cast_or_null<VPWidenRecipe>(V->getDefiningRecipe()); if (!R || R->getOpcode() != WideMember0->getOpcode() || R->getNumOperands() > 2) - return nullptr; + return; if (any_of(enumerate(R->operands()), [WideMember0, Idx = I](const auto &P) { const auto &[OpIdx, OpV] = P; return !canNarrowLoad(WideMember0, OpIdx, OpV, Idx); })) - return nullptr; + return; } StoreGroups.push_back(InterleaveR); } if (StoreGroups.empty()) - return nullptr; - - // All interleave groups in Plan can be narrowed for VFToOptimize. Split the - // original Plan into 2: a) a new clone which contains all VFs of Plan, except - // VFToOptimize, and b) the original Plan with VFToOptimize as single VF. - std::unique_ptr<VPlan> NewPlan; - if (size(Plan.vectorFactors()) != 1) { - NewPlan = std::unique_ptr<VPlan>(Plan.duplicate()); - Plan.setVF(*VFToOptimize); - NewPlan->removeVF(*VFToOptimize); - } + return; // Convert InterleaveGroup \p R to a single VPWidenLoadRecipe. SmallPtrSet<VPValue *, 4> NarrowedOps; @@ -4283,8 +4252,9 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, auto *Inc = cast<VPInstruction>(CanIV->getBackedgeValue()); VPBuilder PHBuilder(Plan.getVectorPreheader()); - VPValue *UF = &Plan.getSymbolicUF(); - if (VFToOptimize->isScalable()) { + VPValue *UF = Plan.getOrAddLiveIn( + ConstantInt::get(CanIV->getScalarType(), 1 * Plan.getUF())); + if (VF.isScalable()) { VPValue *VScale = PHBuilder.createElementCount( CanIV->getScalarType(), ElementCount::getScalable(1)); VPValue *VScaleUF = PHBuilder.createNaryOp(Instruction::Mul, {VScale, UF}); @@ -4296,10 +4266,6 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, Plan.getOrAddLiveIn(ConstantInt::get(CanIV->getScalarType(), 1))); } removeDeadRecipes(Plan); - assert(none_of(*VectorLoop->getEntryBasicBlock(), - IsaPred<VPVectorPointerRecipe>) && - "All VPVectorPointerRecipes should have been removed"); - return NewPlan; } /// Add branch weight metadata, if the \p Plan's middle block is terminated by a diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index ca8d956..b28559b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -341,20 +341,14 @@ struct VPlanTransforms { static DenseMap<const SCEV *, Value *> expandSCEVs(VPlan &Plan, ScalarEvolution &SE); - /// Try to find a single VF among \p Plan's VFs for which all interleave - /// groups (with known minimum VF elements) can be replaced by wide loads and - /// stores processing VF elements, if all transformed interleave groups access - /// the full vector width (checked via the maximum vector register width). If - /// the transformation can be applied, the original \p Plan will be split in - /// 2: - /// 1. The original Plan with the single VF containing the optimized recipes - /// using wide loads instead of interleave groups. - /// 2. A new clone which contains all VFs of Plan except the optimized VF. - /// - /// This effectively is a very simple form of loop-aware SLP, where we use - /// interleave groups to identify candidates. - static std::unique_ptr<VPlan> - narrowInterleaveGroups(VPlan &Plan, const TargetTransformInfo &TTI); + /// Try to convert a plan with interleave groups with VF elements to a plan + /// with the interleave groups replaced by wide loads and stores processing VF + /// elements, if all transformed interleave groups access the full vector + /// width (checked via \o VectorRegWidth). This effectively is a very simple + /// form of loop-aware SLP, where we use interleave groups to identify + /// candidates. + static void narrowInterleaveGroups(VPlan &Plan, ElementCount VF, + unsigned VectorRegWidth); /// Predicate and linearize the control-flow in the only loop region of /// \p Plan. If \p FoldTail is true, create a mask guarding the loop diff --git a/llvm/test/Analysis/BranchProbabilityInfo/pr22718.ll b/llvm/test/Analysis/BranchProbabilityInfo/pr22718.ll index ed851f2..67ce44e 100644 --- a/llvm/test/Analysis/BranchProbabilityInfo/pr22718.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/pr22718.ll @@ -12,14 +12,14 @@ @.str = private unnamed_addr constant [17 x i8] c"x = %lu\0Ay = %lu\0A\00", align 1 ; Function Attrs: inlinehint nounwind uwtable -define i32 @main() #0 { +define i32 @main() { entry: %retval = alloca i32, align 4 %i = alloca i64, align 8 store i32 0, ptr %retval store i64 0, ptr @y, align 8 store i64 0, ptr @x, align 8 - call void @srand(i32 422304) #3 + call void @srand(i32 422304) store i64 0, ptr %i, align 8 br label %for.cond @@ -29,7 +29,7 @@ for.cond: ; preds = %for.inc, %entry br i1 %cmp, label %for.body, label %for.end, !prof !1 for.body: ; preds = %for.cond - %call = call i32 @rand() #3 + %call = call i32 @rand() %conv = sitofp i32 %call to double %mul = fmul double %conv, 1.000000e+02 %div = fdiv double %mul, 0x41E0000000000000 @@ -65,17 +65,12 @@ for.end: ; preds = %for.cond } ; Function Attrs: nounwind -declare void @srand(i32) #1 +declare void @srand(i32) ; Function Attrs: nounwind -declare i32 @rand() #1 +declare i32 @rand() -declare i32 @printf(ptr, ...) #2 - -attributes #0 = { inlinehint nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #3 = { nounwind } +declare i32 @printf(ptr, ...) !llvm.ident = !{!0} diff --git a/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll b/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll index 245e8f7..058370c 100644 --- a/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll +++ b/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll @@ -23,10 +23,10 @@ %"class.llvm::Metadata.306.1758.9986.10470.10954.11438.11922.12406.12890.13374.13858.15310.15794.16278.17730.19182.21118.25958.26926.29346.29830.30314.30798.31282.31766.32250.32734.33702.36606.38058.41638" = type { i8, i8, i16, i32 } ; Function Attrs: argmemonly nounwind -declare void @llvm.lifetime.end(ptr nocapture) #0 +declare void @llvm.lifetime.end(ptr nocapture) ; Function Attrs: nounwind ssp uwtable -define hidden void @fun(ptr %N, i1 %arg) #1 align 2 { +define hidden void @fun(ptr %N, i1 %arg) align 2 { ; CHECK: define entry: %NumOperands.i = getelementptr inbounds %"class.llvm::SDNode.310.1762.9990.10474.10958.11442.11926.12410.12894.13378.13862.15314.15798.16282.17734.19186.21122.25962.26930.29350.29834.30318.30802.31286.31770.32254.32738.33706.36610.38062.41642", ptr %N, i64 0, i32 8 @@ -47,8 +47,6 @@ for.body: ; preds = %for.body, %for.body br i1 %exitcond193, label %for.cond.cleanup, label %for.body } -attributes #0 = { argmemonly nounwind } -attributes #1 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.ident = !{!0} diff --git a/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll b/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll index 0c0fb41..891d604 100644 --- a/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll +++ b/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll @@ -4,7 +4,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" ; Function Attrs: noinline nounwind uwtable -define void @mat_mul(ptr %C, ptr %A, ptr %B, i64 %N) #0 !kernel_arg_addr_space !2 !kernel_arg_access_qual !3 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !5 { +define void @mat_mul(ptr %C, ptr %A, ptr %B, i64 %N) !kernel_arg_addr_space !2 !kernel_arg_access_qual !3 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !5 { ; CHECK-LABEL: 'mat_mul' ; CHECK-NEXT: Inst: %tmp = load float, ptr %arrayidx, align 4 ; CHECK-NEXT: AccessFunction: {(4 * %N * %call),+,4}<%for.inc> @@ -22,8 +22,8 @@ entry: br label %entry.split entry.split: ; preds = %entry - %call = tail call i64 @_Z13get_global_idj(i32 0) #3 - %call1 = tail call i64 @_Z13get_global_idj(i32 1) #3 + %call = tail call i64 @_Z13get_global_idj(i32 0) + %call1 = tail call i64 @_Z13get_global_idj(i32 1) %cmp1 = icmp sgt i64 %N, 0 %mul = mul nsw i64 %call, %N br i1 %cmp1, label %for.inc.lr.ph, label %for.end @@ -59,15 +59,10 @@ for.end: ; preds = %for.cond.for.end_cr } ; Function Attrs: nounwind readnone -declare i64 @_Z13get_global_idj(i32) #1 +declare i64 @_Z13get_global_idj(i32) ; Function Attrs: nounwind readnone speculatable -declare float @llvm.fmuladd.f32(float, float, float) #2 - -attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #2 = { nounwind readnone speculatable } -attributes #3 = { nounwind readnone } +declare float @llvm.fmuladd.f32(float, float, float) !llvm.module.flags = !{!0} !llvm.ident = !{!1} diff --git a/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll b/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll index b498d7064..f5be89a 100644 --- a/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll +++ b/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll @@ -30,7 +30,7 @@ target datalayout = "e-m:e-p:32:32-i1:32-i64:64-a:0-v32:32-n16:32" %20 = type { [768 x i32] } %21 = type { [416 x i32] } -define void @test(ptr %A, ptr %B, i1 %arg, i32 %n, i32 %m) #0 align 2 { +define void @test(ptr %A, ptr %B, i1 %arg, i32 %n, i32 %m) align 2 { ; CHECK-LABEL: 'test' ; CHECK-NEXT: Src: %v1 = load i32, ptr %B, align 4 --> Dst: %v1 = load i32, ptr %B, align 4 ; CHECK-NEXT: da analyze - none! @@ -91,5 +91,3 @@ bb38: bb40: ret void } - -attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } diff --git a/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll b/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll index e5d5d21e..eba017a 100644 --- a/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll +++ b/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll @@ -52,7 +52,7 @@ for.end: @a = global [10004 x [10004 x i32]] zeroinitializer, align 16 ; Function Attrs: nounwind uwtable -define void @coupled_miv_type_mismatch(i32 %n) #0 { +define void @coupled_miv_type_mismatch(i32 %n) { ; CHECK-LABEL: 'coupled_miv_type_mismatch' ; CHECK-NEXT: Src: %2 = load i32, ptr %arrayidx5, align 4 --> Dst: %2 = load i32, ptr %arrayidx5, align 4 ; CHECK-NEXT: da analyze - none! @@ -101,8 +101,6 @@ for.end13: ; preds = %for.cond ret void } -attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.ident = !{!0} !0 = !{!"clang version 3.7.0"} diff --git a/llvm/test/Analysis/MemoryDependenceAnalysis/invariant.group-bug.ll b/llvm/test/Analysis/MemoryDependenceAnalysis/invariant.group-bug.ll index c11191e..5470ef9 100644 --- a/llvm/test/Analysis/MemoryDependenceAnalysis/invariant.group-bug.ll +++ b/llvm/test/Analysis/MemoryDependenceAnalysis/invariant.group-bug.ll @@ -17,13 +17,13 @@ target triple = "x86_64-grtev4-linux-gnu" %4 = type { ptr } %5 = type { i64, [8 x i8] } -define void @fail(ptr noalias sret(i1) %arg, ptr %arg1, ptr %arg2, ptr %arg3, i1 %arg4) local_unnamed_addr #0 { +define void @fail(ptr noalias sret(i1) %arg, ptr %arg1, ptr %arg2, ptr %arg3, i1 %arg4) local_unnamed_addr { ; CHECK-LABEL: @fail( ; CHECK-NEXT: bb: ; CHECK-NEXT: [[I4:%.*]] = load ptr, ptr [[ARG1:%.*]], align 8, !invariant.group [[META6:![0-9]+]] ; CHECK-NEXT: [[I5:%.*]] = getelementptr inbounds ptr, ptr [[I4]], i64 6 ; CHECK-NEXT: [[I6:%.*]] = load ptr, ptr [[I5]], align 8, !invariant.load [[META6]] -; CHECK-NEXT: [[I7:%.*]] = tail call i64 [[I6]](ptr [[ARG1]]) #[[ATTR1:[0-9]+]] +; CHECK-NEXT: [[I7:%.*]] = tail call i64 [[I6]](ptr [[ARG1]]) ; CHECK-NEXT: [[I9:%.*]] = load ptr, ptr [[ARG2:%.*]], align 8 ; CHECK-NEXT: store i8 0, ptr [[I9]], align 1 ; CHECK-NEXT: br i1 [[ARG4:%.*]], label [[BB10:%.*]], label [[BB29:%.*]] @@ -32,7 +32,7 @@ define void @fail(ptr noalias sret(i1) %arg, ptr %arg1, ptr %arg2, ptr %arg3, i1 ; CHECK-NEXT: [[I15_PRE:%.*]] = load ptr, ptr [[I14_PHI_TRANS_INSERT]], align 8, !invariant.load [[META6]] ; CHECK-NEXT: br label [[BB12:%.*]] ; CHECK: bb12: -; CHECK-NEXT: [[I16:%.*]] = call i64 [[I15_PRE]](ptr nonnull [[ARG1]], ptr null, i64 0) #[[ATTR1]] +; CHECK-NEXT: [[I16:%.*]] = call i64 [[I15_PRE]](ptr nonnull [[ARG1]], ptr null, i64 0) ; CHECK-NEXT: br i1 true, label [[BB28:%.*]], label [[BB17:%.*]] ; CHECK: bb17: ; CHECK-NEXT: br i1 true, label [[BB18:%.*]], label [[BB21:%.*]] @@ -55,7 +55,7 @@ bb: %i4 = load ptr, ptr %arg1, align 8, !invariant.group !6 %i5 = getelementptr inbounds ptr, ptr %i4, i64 6 %i6 = load ptr, ptr %i5, align 8, !invariant.load !6 - %i7 = tail call i64 %i6(ptr %arg1) #1 + %i7 = tail call i64 %i6(ptr %arg1) %i9 = load ptr, ptr %arg2, align 8 store i8 0, ptr %i9, align 1 br i1 %arg4, label %bb10, label %bb29 @@ -67,7 +67,7 @@ bb12: ; preds = %bb28, %bb10 %i13 = load ptr, ptr %arg1, align 8, !invariant.group !6 %i14 = getelementptr inbounds ptr, ptr %i13, i64 22 %i15 = load ptr, ptr %i14, align 8, !invariant.load !6 - %i16 = call i64 %i15(ptr nonnull %arg1, ptr null, i64 0) #1 + %i16 = call i64 %i15(ptr nonnull %arg1, ptr null, i64 0) br i1 %arg4, label %bb28, label %bb17 bb17: ; preds = %bb12 @@ -110,9 +110,6 @@ bb29: ; preds = %bb28, %bb ret void } -attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="non-leaf" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="non-leaf" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.linker.options = !{} !llvm.module.flags = !{!0, !1, !3, !4, !5} diff --git a/llvm/test/Analysis/MemorySSA/pr28880.ll b/llvm/test/Analysis/MemorySSA/pr28880.ll index 98f3261..a2690b9 100644 --- a/llvm/test/Analysis/MemorySSA/pr28880.ll +++ b/llvm/test/Analysis/MemorySSA/pr28880.ll @@ -8,7 +8,7 @@ @global.1 = external hidden unnamed_addr global double, align 8 ; Function Attrs: nounwind ssp uwtable -define hidden fastcc void @hoge(i1 %arg) unnamed_addr #0 { +define hidden fastcc void @hoge(i1 %arg) unnamed_addr { bb: br i1 %arg, label %bb1, label %bb2 @@ -45,6 +45,3 @@ bb4: ; preds = %bb3 bb6: ; preds = %bb3 unreachable } - -attributes #0 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" } - diff --git a/llvm/test/Analysis/MemorySSA/pr39197.ll b/llvm/test/Analysis/MemorySSA/pr39197.ll index af57b3c..6be0c58 100644 --- a/llvm/test/Analysis/MemorySSA/pr39197.ll +++ b/llvm/test/Analysis/MemorySSA/pr39197.ll @@ -12,13 +12,13 @@ declare void @dummy() ; CHECK-LABEL: @main() ; Function Attrs: nounwind -define dso_local void @main() #0 { +define dso_local void @main() { call void @func_1() unreachable } ; Function Attrs: nounwind -define dso_local void @func_1() #0 { +define dso_local void @func_1() { %1 = alloca ptr, align 8 %2 = call signext i32 @func_2() %3 = icmp ne i32 %2, 0 @@ -64,45 +64,45 @@ define dso_local void @func_1() #0 { } ; Function Attrs: nounwind -declare dso_local signext i32 @func_2() #0 +declare dso_local signext i32 @func_2() ; Function Attrs: nounwind -define dso_local void @safe_sub_func_uint8_t_u_u() #0 { +define dso_local void @safe_sub_func_uint8_t_u_u() { ret void } ; Function Attrs: nounwind -define dso_local void @safe_add_func_int64_t_s_s() #0 { +define dso_local void @safe_add_func_int64_t_s_s() { ret void } ; Function Attrs: nounwind -define dso_local void @safe_rshift_func_int16_t_s_u() #0 { +define dso_local void @safe_rshift_func_int16_t_s_u() { ret void } ; Function Attrs: nounwind -define dso_local void @safe_div_func_uint8_t_u_u() #0 { +define dso_local void @safe_div_func_uint8_t_u_u() { ret void } ; Function Attrs: nounwind -define dso_local void @safe_mul_func_uint16_t_u_u() #0 { +define dso_local void @safe_mul_func_uint16_t_u_u() { ret void } ; Function Attrs: nounwind -define dso_local void @safe_mul_func_int16_t_s_s() #0 { +define dso_local void @safe_mul_func_int16_t_s_s() { ret void } ; Function Attrs: nounwind -define dso_local void @safe_div_func_int32_t_s_s() #0 { +define dso_local void @safe_div_func_int32_t_s_s() { ret void } ; Function Attrs: nounwind -define dso_local signext i16 @safe_sub_func_int16_t_s_s(i16 signext) #0 { +define dso_local signext i16 @safe_sub_func_int16_t_s_s(i16 signext) { %2 = alloca i16, align 2 store i16 %0, ptr %2, align 2, !tbaa !1 %3 = load i16, ptr %2, align 2, !tbaa !1 @@ -113,29 +113,25 @@ define dso_local signext i16 @safe_sub_func_int16_t_s_s(i16 signext) #0 { } ; Function Attrs: nounwind -define dso_local void @safe_add_func_uint16_t_u_u() #0 { +define dso_local void @safe_add_func_uint16_t_u_u() { ret void } ; Function Attrs: nounwind -define dso_local void @safe_div_func_int8_t_s_s() #0 { +define dso_local void @safe_div_func_int8_t_s_s() { ret void } ; Function Attrs: nounwind -define dso_local void @safe_add_func_int16_t_s_s() #0 { +define dso_local void @safe_add_func_int16_t_s_s() { ret void } ; Function Attrs: nounwind -define dso_local void @safe_add_func_uint8_t_u_u() #0 { +define dso_local void @safe_add_func_uint8_t_u_u() { ret void } -attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { argmemonly nounwind } -attributes #2 = { nounwind } - !llvm.ident = !{!0} !0 = !{!"clang version 8.0.0 (http://llvm.org/git/clang.git 7cda4756fc9713d98fd3513b8df172700f267bad) (http://llvm.org/git/llvm.git 199c0d32e96b646bd8cf6beeaf0f99f8a434b56a)"} diff --git a/llvm/test/Analysis/MemorySSA/pr40038.ll b/llvm/test/Analysis/MemorySSA/pr40038.ll index efdcbe5..39ea78b 100644 --- a/llvm/test/Analysis/MemorySSA/pr40038.ll +++ b/llvm/test/Analysis/MemorySSA/pr40038.ll @@ -10,21 +10,21 @@ target triple = "s390x-ibm-linux" ; Function Attrs: nounwind ; CHECK-LABEL: @main -define dso_local void @main() #0 { +define dso_local void @main() { bb: call void @func_1() unreachable } ; Function Attrs: nounwind -define dso_local void @func_1() #0 { +define dso_local void @func_1() { bb: call void @func_2() unreachable } ; Function Attrs: nounwind -define dso_local void @func_2() #0 { +define dso_local void @func_2() { bb: %tmp = alloca i32, align 4 store i32 0, ptr @g_80, align 4, !tbaa !1 @@ -68,10 +68,7 @@ bb18: ; preds = %bb12, %bb1 } ; Function Attrs: cold noreturn nounwind -declare void @llvm.trap() #1 - -attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="z13" "target-features"="+transactional-execution,+vector" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { cold noreturn nounwind } +declare void @llvm.trap() !llvm.ident = !{!0} diff --git a/llvm/test/Analysis/MemorySSA/pr43569.ll b/llvm/test/Analysis/MemorySSA/pr43569.ll index 02d074e..c81f8d4 100644 --- a/llvm/test/Analysis/MemorySSA/pr43569.ll +++ b/llvm/test/Analysis/MemorySSA/pr43569.ll @@ -10,7 +10,7 @@ target triple = "x86_64-unknown-linux-gnu" ; CHECK-LABEL: @c() ; Function Attrs: nounwind uwtable -define dso_local void @c() #0 { +define dso_local void @c() { entry: call void @llvm.instrprof.increment(ptr @__profn_c, i64 68269137, i32 3, i32 0) br label %for.cond @@ -42,8 +42,4 @@ for.end: ; preds = %for.cond1 } ; Function Attrs: nounwind -declare void @llvm.instrprof.increment(ptr, i64, i32, i32) #1 - -attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind } - +declare void @llvm.instrprof.increment(ptr, i64, i32, i32) diff --git a/llvm/test/Analysis/ScalarEvolution/pr22674.ll b/llvm/test/Analysis/ScalarEvolution/pr22674.ll index 95f96ca..b2f4ae6 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr22674.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr22674.ll @@ -11,7 +11,7 @@ target triple = "x86_64-pc-linux-gnux32" %"class.llvm::AttributeImpl.2.1802.3601.5914.6685.7456.8227.9255.9769.10026.18508" = type <{ ptr, %"class.llvm::FoldingSetImpl::Node.1.1801.3600.5913.6684.7455.8226.9254.9768.10025.18505", i8, [3 x i8] }> ; Function Attrs: nounwind uwtable -define void @_ZNK4llvm11AttrBuilder13hasAttributesENS_12AttributeSetEy(i1 %arg) #0 align 2 { +define void @_ZNK4llvm11AttrBuilder13hasAttributesENS_12AttributeSetEy(i1 %arg) align 2 { entry: br i1 %arg, label %cond.false, label %_ZNK4llvm12AttributeSet11getNumSlotsEv.exit @@ -82,8 +82,6 @@ return: ; preds = %_ZNK4llvm9Attribute ret void } -attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.module.flags = !{!0} !llvm.ident = !{!1} diff --git a/llvm/test/Analysis/ScalarEvolution/scev-canonical-mode.ll b/llvm/test/Analysis/ScalarEvolution/scev-canonical-mode.ll index 3879b2e7..d9cc3e5 100644 --- a/llvm/test/Analysis/ScalarEvolution/scev-canonical-mode.ll +++ b/llvm/test/Analysis/ScalarEvolution/scev-canonical-mode.ll @@ -6,7 +6,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" ; Function Attrs: norecurse nounwind uwtable -define void @ehF(i1 %arg) #0 { +define void @ehF(i1 %arg) { entry: br i1 %arg, label %if.then.i, label %hup.exit @@ -28,5 +28,3 @@ for.body.i: ; preds = %for.body.i, %for.bo hup.exit: ; preds = %for.body.i, %if.then.i, %entry ret void } - -attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/PR17620.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/PR17620.ll index 67d81e7..7fb4231 100644 --- a/llvm/test/Analysis/TypeBasedAliasAnalysis/PR17620.ll +++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/PR17620.ll @@ -13,7 +13,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 %classD = type { ptr } ; Function Attrs: ssp uwtable -define ptr @test(ptr %this, ptr %p1) #0 align 2 { +define ptr @test(ptr %this, ptr %p1) align 2 { entry: ; CHECK-LABEL: @test ; CHECK: load ptr, ptr %p1, align 8, !tbaa @@ -25,10 +25,7 @@ entry: unreachable } -declare void @callee(ptr, ptr) #1 - -attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +declare void @callee(ptr, ptr) !llvm.ident = !{!0} diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll index 942fdf5..f9a2988 100644 --- a/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll +++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll @@ -9,7 +9,7 @@ %struct.StructC = type { i16, %struct.StructB, i32 } %struct.StructD = type { i16, %struct.StructB, i32, i8 } -define i32 @_Z1gPjP7StructAy(ptr %s, ptr %A, i64 %count) #0 { +define i32 @_Z1gPjP7StructAy(ptr %s, ptr %A, i64 %count) { entry: ; Access to ptr and &(A->f32). ; CHECK: Function @@ -35,7 +35,7 @@ entry: ret i32 %3 } -define i32 @_Z2g2PjP7StructAy(ptr %s, ptr %A, i64 %count) #0 { +define i32 @_Z2g2PjP7StructAy(ptr %s, ptr %A, i64 %count) { entry: ; Access to ptr and &(A->f16). ; CHECK: Function @@ -60,7 +60,7 @@ entry: ret i32 %3 } -define i32 @_Z2g3P7StructAP7StructBy(ptr %A, ptr %B, i64 %count) #0 { +define i32 @_Z2g3P7StructAP7StructBy(ptr %A, ptr %B, i64 %count) { entry: ; Access to &(A->f32) and &(B->a.f32). ; CHECK: Function @@ -89,7 +89,7 @@ entry: ret i32 %3 } -define i32 @_Z2g4P7StructAP7StructBy(ptr %A, ptr %B, i64 %count) #0 { +define i32 @_Z2g4P7StructAP7StructBy(ptr %A, ptr %B, i64 %count) { entry: ; Access to &(A->f32) and &(B->a.f16). ; CHECK: Function @@ -117,7 +117,7 @@ entry: ret i32 %3 } -define i32 @_Z2g5P7StructAP7StructBy(ptr %A, ptr %B, i64 %count) #0 { +define i32 @_Z2g5P7StructAP7StructBy(ptr %A, ptr %B, i64 %count) { entry: ; Access to &(A->f32) and &(B->f32). ; CHECK: Function @@ -145,7 +145,7 @@ entry: ret i32 %3 } -define i32 @_Z2g6P7StructAP7StructBy(ptr %A, ptr %B, i64 %count) #0 { +define i32 @_Z2g6P7StructAP7StructBy(ptr %A, ptr %B, i64 %count) { entry: ; Access to &(A->f32) and &(B->a.f32_2). ; CHECK: Function @@ -174,7 +174,7 @@ entry: ret i32 %3 } -define i32 @_Z2g7P7StructAP7StructSy(ptr %A, ptr %S, i64 %count) #0 { +define i32 @_Z2g7P7StructAP7StructSy(ptr %A, ptr %S, i64 %count) { entry: ; Access to &(A->f32) and &(S->f32). ; CHECK: Function @@ -202,7 +202,7 @@ entry: ret i32 %3 } -define i32 @_Z2g8P7StructAP7StructSy(ptr %A, ptr %S, i64 %count) #0 { +define i32 @_Z2g8P7StructAP7StructSy(ptr %A, ptr %S, i64 %count) { entry: ; Access to &(A->f32) and &(S->f16). ; CHECK: Function @@ -229,7 +229,7 @@ entry: ret i32 %3 } -define i32 @_Z2g9P7StructSP8StructS2y(ptr %S, ptr %S2, i64 %count) #0 { +define i32 @_Z2g9P7StructSP8StructS2y(ptr %S, ptr %S2, i64 %count) { entry: ; Access to &(S->f32) and &(S2->f32). ; CHECK: Function @@ -257,7 +257,7 @@ entry: ret i32 %3 } -define i32 @_Z3g10P7StructSP8StructS2y(ptr %S, ptr %S2, i64 %count) #0 { +define i32 @_Z3g10P7StructSP8StructS2y(ptr %S, ptr %S2, i64 %count) { entry: ; Access to &(S->f32) and &(S2->f16). ; CHECK: Function @@ -284,7 +284,7 @@ entry: ret i32 %3 } -define i32 @_Z3g11P7StructCP7StructDy(ptr %C, ptr %D, i64 %count) #0 { +define i32 @_Z3g11P7StructCP7StructDy(ptr %C, ptr %D, i64 %count) { entry: ; Access to &(C->b.a.f32) and &(D->b.a.f32). ; CHECK: Function @@ -318,7 +318,7 @@ entry: ret i32 %3 } -define i32 @_Z3g12P7StructCP7StructDy(ptr %C, ptr %D, i64 %count) #0 { +define i32 @_Z3g12P7StructCP7StructDy(ptr %C, ptr %D, i64 %count) { entry: ; Access to &(b1->a.f32) and &(b2->a.f32). ; CHECK: Function @@ -357,8 +357,6 @@ entry: ret i32 %5 } -attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } - !0 = !{!1, !1, i64 0} !1 = !{!"any pointer", !2} !2 = !{!"omnipotent char", !3} diff --git a/llvm/test/CodeGen/AArch64/pr164181.ll b/llvm/test/CodeGen/AArch64/pr164181.ll new file mode 100644 index 0000000..4ec63ec --- /dev/null +++ b/llvm/test/CodeGen/AArch64/pr164181.ll @@ -0,0 +1,640 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -verify-machineinstrs < %s | FileCheck %s + +; This test recreates a regalloc crash reported in +; https://github.com/llvm/llvm-project/issues/164181 +; When rematting an instruction we need to make sure to constrain the newly +; allocated register to both the rematted def's reg class and the use's reg +; class. + +target triple = "aarch64-unknown-linux-gnu" + +@var_32 = external global i16 +@var_35 = external global i64 +@var_39 = external global i64 +@var_46 = external global i64 +@var_50 = external global i32 + +define void @f(i1 %var_0, i16 %var_1, i64 %var_2, i8 %var_3, i16 %var_4, i1 %var_5, i32 %var_6, i32 %var_7, i8 %var_10, i64 %var_11, i8 %var_14, i32 %var_15, i64 %var_16, ptr %arr_3, ptr %arr_4, ptr %arr_6, ptr %arr_7, ptr %arr_12, ptr %arr_13, ptr %arr_19, i64 %mul, i64 %conv35, i64 %idxprom138.us16, i8 %0, i8 %1, ptr %invariant.gep875.us) #0 { +; CHECK-LABEL: f: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sub sp, sp, #240 +; CHECK-NEXT: str x30, [sp, #144] // 8-byte Folded Spill +; CHECK-NEXT: stp x28, x27, [sp, #160] // 16-byte Folded Spill +; CHECK-NEXT: stp x26, x25, [sp, #176] // 16-byte Folded Spill +; CHECK-NEXT: stp x24, x23, [sp, #192] // 16-byte Folded Spill +; CHECK-NEXT: stp x22, x21, [sp, #208] // 16-byte Folded Spill +; CHECK-NEXT: stp x20, x19, [sp, #224] // 16-byte Folded Spill +; CHECK-NEXT: str w6, [sp, #20] // 4-byte Folded Spill +; CHECK-NEXT: str w4, [sp, #72] // 4-byte Folded Spill +; CHECK-NEXT: str w3, [sp, #112] // 4-byte Folded Spill +; CHECK-NEXT: str w5, [sp, #36] // 4-byte Folded Spill +; CHECK-NEXT: tbz w5, #0, .LBB0_43 +; CHECK-NEXT: // %bb.1: // %for.body41.lr.ph +; CHECK-NEXT: ldr x4, [sp, #312] +; CHECK-NEXT: ldr x14, [sp, #280] +; CHECK-NEXT: tbz w0, #0, .LBB0_42 +; CHECK-NEXT: // %bb.2: // %for.body41.us.preheader +; CHECK-NEXT: ldrb w8, [sp, #368] +; CHECK-NEXT: ldrb w12, [sp, #256] +; CHECK-NEXT: ldr w26, [sp, #264] +; CHECK-NEXT: adrp x20, :got:var_50 +; CHECK-NEXT: mov x28, #-1 // =0xffffffffffffffff +; CHECK-NEXT: mov w21, #36006 // =0x8ca6 +; CHECK-NEXT: ldr x11, [sp, #376] +; CHECK-NEXT: ldrb w13, [sp, #360] +; CHECK-NEXT: ldp x17, x16, [sp, #296] +; CHECK-NEXT: mov w22, #1 // =0x1 +; CHECK-NEXT: add x27, x14, #120 +; CHECK-NEXT: ldr x18, [sp, #288] +; CHECK-NEXT: ldr x7, [sp, #272] +; CHECK-NEXT: ldr x5, [sp, #248] +; CHECK-NEXT: mov x10, xzr +; CHECK-NEXT: mov w23, wzr +; CHECK-NEXT: mov w30, wzr +; CHECK-NEXT: ldrb w19, [sp, #240] +; CHECK-NEXT: mov w25, wzr +; CHECK-NEXT: mov x24, xzr +; CHECK-NEXT: str w8, [sp, #108] // 4-byte Folded Spill +; CHECK-NEXT: mov x3, x26 +; CHECK-NEXT: ldp x9, x8, [sp, #344] +; CHECK-NEXT: str w12, [sp, #92] // 4-byte Folded Spill +; CHECK-NEXT: mov w12, #1 // =0x1 +; CHECK-NEXT: bic w12, w12, w0 +; CHECK-NEXT: str w12, [sp, #76] // 4-byte Folded Spill +; CHECK-NEXT: mov w12, #48 // =0x30 +; CHECK-NEXT: str x9, [sp, #136] // 8-byte Folded Spill +; CHECK-NEXT: ldp x9, x15, [sp, #328] +; CHECK-NEXT: madd x8, x8, x12, x9 +; CHECK-NEXT: str x8, [sp, #64] // 8-byte Folded Spill +; CHECK-NEXT: add x8, x26, w26, uxtw #1 +; CHECK-NEXT: ldr x20, [x20, :got_lo12:var_50] +; CHECK-NEXT: str x26, [sp, #96] // 8-byte Folded Spill +; CHECK-NEXT: str x14, [sp, #152] // 8-byte Folded Spill +; CHECK-NEXT: lsl x6, x8, #3 +; CHECK-NEXT: add x8, x14, #120 +; CHECK-NEXT: str x4, [sp, #24] // 8-byte Folded Spill +; CHECK-NEXT: str w19, [sp, #16] // 4-byte Folded Spill +; CHECK-NEXT: str x8, [sp, #80] // 8-byte Folded Spill +; CHECK-NEXT: b .LBB0_4 +; CHECK-NEXT: .p2align 5, , 16 +; CHECK-NEXT: .LBB0_3: // in Loop: Header=BB0_4 Depth=1 +; CHECK-NEXT: ldr w19, [sp, #16] // 4-byte Folded Reload +; CHECK-NEXT: ldr x24, [sp, #40] // 8-byte Folded Reload +; CHECK-NEXT: ldr x14, [sp, #152] // 8-byte Folded Reload +; CHECK-NEXT: mov w23, #1 // =0x1 +; CHECK-NEXT: mov w30, #1 // =0x1 +; CHECK-NEXT: mov w25, w19 +; CHECK-NEXT: .LBB0_4: // %for.body41.us +; CHECK-NEXT: // =>This Loop Header: Depth=1 +; CHECK-NEXT: // Child Loop BB0_6 Depth 2 +; CHECK-NEXT: // Child Loop BB0_8 Depth 3 +; CHECK-NEXT: // Child Loop BB0_10 Depth 4 +; CHECK-NEXT: // Child Loop BB0_11 Depth 5 +; CHECK-NEXT: // Child Loop BB0_28 Depth 5 +; CHECK-NEXT: // Child Loop BB0_39 Depth 5 +; CHECK-NEXT: ldr w8, [sp, #20] // 4-byte Folded Reload +; CHECK-NEXT: mov x12, x24 +; CHECK-NEXT: str x24, [sp, #48] // 8-byte Folded Spill +; CHECK-NEXT: str w8, [x14] +; CHECK-NEXT: mov w8, #1 // =0x1 +; CHECK-NEXT: strb w19, [x14] +; CHECK-NEXT: b .LBB0_6 +; CHECK-NEXT: .p2align 5, , 16 +; CHECK-NEXT: .LBB0_5: // %for.cond.cleanup93.us +; CHECK-NEXT: // in Loop: Header=BB0_6 Depth=2 +; CHECK-NEXT: ldr w9, [sp, #36] // 4-byte Folded Reload +; CHECK-NEXT: ldr x4, [sp, #24] // 8-byte Folded Reload +; CHECK-NEXT: ldp x24, x12, [sp, #48] // 16-byte Folded Reload +; CHECK-NEXT: mov x22, xzr +; CHECK-NEXT: mov w25, wzr +; CHECK-NEXT: mov w8, wzr +; CHECK-NEXT: tbz w9, #0, .LBB0_3 +; CHECK-NEXT: .LBB0_6: // %for.body67.us +; CHECK-NEXT: // Parent Loop BB0_4 Depth=1 +; CHECK-NEXT: // => This Loop Header: Depth=2 +; CHECK-NEXT: // Child Loop BB0_8 Depth 3 +; CHECK-NEXT: // Child Loop BB0_10 Depth 4 +; CHECK-NEXT: // Child Loop BB0_11 Depth 5 +; CHECK-NEXT: // Child Loop BB0_28 Depth 5 +; CHECK-NEXT: // Child Loop BB0_39 Depth 5 +; CHECK-NEXT: str x12, [sp, #40] // 8-byte Folded Spill +; CHECK-NEXT: cmn x24, #30 +; CHECK-NEXT: mov x12, #-30 // =0xffffffffffffffe2 +; CHECK-NEXT: add x19, x4, w8, sxtw #2 +; CHECK-NEXT: mov x9, xzr +; CHECK-NEXT: csel x12, x24, x12, lo +; CHECK-NEXT: mov w4, w30 +; CHECK-NEXT: str x12, [sp, #56] // 8-byte Folded Spill +; CHECK-NEXT: b .LBB0_8 +; CHECK-NEXT: .p2align 5, , 16 +; CHECK-NEXT: .LBB0_7: // %for.cond.cleanup98.us +; CHECK-NEXT: // in Loop: Header=BB0_8 Depth=3 +; CHECK-NEXT: ldr w4, [sp, #72] // 4-byte Folded Reload +; CHECK-NEXT: ldr w23, [sp, #128] // 4-byte Folded Reload +; CHECK-NEXT: mov w9, #1 // =0x1 +; CHECK-NEXT: mov x22, xzr +; CHECK-NEXT: tbnz w0, #0, .LBB0_5 +; CHECK-NEXT: .LBB0_8: // %for.cond95.preheader.us +; CHECK-NEXT: // Parent Loop BB0_4 Depth=1 +; CHECK-NEXT: // Parent Loop BB0_6 Depth=2 +; CHECK-NEXT: // => This Loop Header: Depth=3 +; CHECK-NEXT: // Child Loop BB0_10 Depth 4 +; CHECK-NEXT: // Child Loop BB0_11 Depth 5 +; CHECK-NEXT: // Child Loop BB0_28 Depth 5 +; CHECK-NEXT: // Child Loop BB0_39 Depth 5 +; CHECK-NEXT: ldr x8, [sp, #64] // 8-byte Folded Reload +; CHECK-NEXT: mov w14, #1152 // =0x480 +; CHECK-NEXT: mov w24, #1 // =0x1 +; CHECK-NEXT: mov w12, wzr +; CHECK-NEXT: str wzr, [sp, #132] // 4-byte Folded Spill +; CHECK-NEXT: mov w30, w4 +; CHECK-NEXT: madd x8, x9, x14, x8 +; CHECK-NEXT: mov w14, #1 // =0x1 +; CHECK-NEXT: str x8, [sp, #120] // 8-byte Folded Spill +; CHECK-NEXT: add x8, x9, x9, lsl #1 +; CHECK-NEXT: lsl x26, x8, #4 +; CHECK-NEXT: sxtb w8, w23 +; CHECK-NEXT: mov w23, w25 +; CHECK-NEXT: str w8, [sp, #116] // 4-byte Folded Spill +; CHECK-NEXT: b .LBB0_10 +; CHECK-NEXT: .p2align 5, , 16 +; CHECK-NEXT: .LBB0_9: // %for.cond510.preheader.us +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: ldr w23, [sp, #92] // 4-byte Folded Reload +; CHECK-NEXT: mov x22, x8 +; CHECK-NEXT: ldr x3, [sp, #96] // 8-byte Folded Reload +; CHECK-NEXT: ldr x27, [sp, #80] // 8-byte Folded Reload +; CHECK-NEXT: mov x28, #-1 // =0xffffffffffffffff +; CHECK-NEXT: mov x14, xzr +; CHECK-NEXT: ldr w8, [sp, #76] // 4-byte Folded Reload +; CHECK-NEXT: tbz w8, #31, .LBB0_7 +; CHECK-NEXT: .LBB0_10: // %for.body99.us +; CHECK-NEXT: // Parent Loop BB0_4 Depth=1 +; CHECK-NEXT: // Parent Loop BB0_6 Depth=2 +; CHECK-NEXT: // Parent Loop BB0_8 Depth=3 +; CHECK-NEXT: // => This Loop Header: Depth=4 +; CHECK-NEXT: // Child Loop BB0_11 Depth 5 +; CHECK-NEXT: // Child Loop BB0_28 Depth 5 +; CHECK-NEXT: // Child Loop BB0_39 Depth 5 +; CHECK-NEXT: ldr w8, [sp, #116] // 4-byte Folded Reload +; CHECK-NEXT: and w8, w8, w8, asr #31 +; CHECK-NEXT: str w8, [sp, #128] // 4-byte Folded Spill +; CHECK-NEXT: .p2align 5, , 16 +; CHECK-NEXT: .LBB0_11: // %for.body113.us +; CHECK-NEXT: // Parent Loop BB0_4 Depth=1 +; CHECK-NEXT: // Parent Loop BB0_6 Depth=2 +; CHECK-NEXT: // Parent Loop BB0_8 Depth=3 +; CHECK-NEXT: // Parent Loop BB0_10 Depth=4 +; CHECK-NEXT: // => This Inner Loop Header: Depth=5 +; CHECK-NEXT: tbnz w0, #0, .LBB0_11 +; CHECK-NEXT: // %bb.12: // %for.cond131.preheader.us +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: ldr w8, [sp, #112] // 4-byte Folded Reload +; CHECK-NEXT: mov w4, #1 // =0x1 +; CHECK-NEXT: strb w8, [x18] +; CHECK-NEXT: ldr x8, [sp, #120] // 8-byte Folded Reload +; CHECK-NEXT: ldrh w8, [x8] +; CHECK-NEXT: cbnz w4, .LBB0_14 +; CHECK-NEXT: // %bb.13: // %cond.true146.us +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: ldrsb w4, [x27, x3] +; CHECK-NEXT: b .LBB0_15 +; CHECK-NEXT: .p2align 5, , 16 +; CHECK-NEXT: .LBB0_14: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: mov w4, wzr +; CHECK-NEXT: .LBB0_15: // %cond.end154.us +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: mov w25, #18984 // =0x4a28 +; CHECK-NEXT: mul w8, w8, w25 +; CHECK-NEXT: and w8, w8, #0xfff8 +; CHECK-NEXT: lsl w8, w8, w4 +; CHECK-NEXT: cbz w8, .LBB0_17 +; CHECK-NEXT: // %bb.16: // %if.then.us +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: str wzr, [sp, #132] // 4-byte Folded Spill +; CHECK-NEXT: str wzr, [x18] +; CHECK-NEXT: .LBB0_17: // %if.end.us +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: ldr w8, [sp, #108] // 4-byte Folded Reload +; CHECK-NEXT: mov w4, #18984 // =0x4a28 +; CHECK-NEXT: mov w25, w23 +; CHECK-NEXT: strb w8, [x18] +; CHECK-NEXT: ldrsb w8, [x27, x3] +; CHECK-NEXT: lsl w8, w4, w8 +; CHECK-NEXT: mov x4, #-18403 // =0xffffffffffffb81d +; CHECK-NEXT: movk x4, #58909, lsl #16 +; CHECK-NEXT: cbz w8, .LBB0_19 +; CHECK-NEXT: // %bb.18: // %if.then.us.2 +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: str wzr, [sp, #132] // 4-byte Folded Spill +; CHECK-NEXT: strb wzr, [x18] +; CHECK-NEXT: .LBB0_19: // %if.then.us.5 +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: ldr w23, [sp, #132] // 4-byte Folded Reload +; CHECK-NEXT: mov w8, #29625 // =0x73b9 +; CHECK-NEXT: movk w8, #21515, lsl #16 +; CHECK-NEXT: cmp w23, w8 +; CHECK-NEXT: csel w23, w23, w8, lt +; CHECK-NEXT: str w23, [sp, #132] // 4-byte Folded Spill +; CHECK-NEXT: tbz w0, #0, .LBB0_21 +; CHECK-NEXT: // %bb.20: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: mov w8, wzr +; CHECK-NEXT: b .LBB0_22 +; CHECK-NEXT: .p2align 5, , 16 +; CHECK-NEXT: .LBB0_21: // %cond.true146.us.7 +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: ldrsb w8, [x27, x3] +; CHECK-NEXT: .LBB0_22: // %cond.end154.us.7 +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: mov w23, #18984 // =0x4a28 +; CHECK-NEXT: mov w3, #149 // =0x95 +; CHECK-NEXT: lsl w8, w23, w8 +; CHECK-NEXT: cbz w8, .LBB0_24 +; CHECK-NEXT: // %bb.23: // %if.then.us.7 +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: ldr x8, [sp, #152] // 8-byte Folded Reload +; CHECK-NEXT: str wzr, [sp, #132] // 4-byte Folded Spill +; CHECK-NEXT: str wzr, [x8] +; CHECK-NEXT: .LBB0_24: // %if.end.us.7 +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: mov x23, xzr +; CHECK-NEXT: b .LBB0_28 +; CHECK-NEXT: .p2align 5, , 16 +; CHECK-NEXT: .LBB0_25: // %cond.true331.us +; CHECK-NEXT: // in Loop: Header=BB0_28 Depth=5 +; CHECK-NEXT: ldrsb w4, [x10] +; CHECK-NEXT: .LBB0_26: // %cond.end345.us +; CHECK-NEXT: // in Loop: Header=BB0_28 Depth=5 +; CHECK-NEXT: strh w4, [x18] +; CHECK-NEXT: mul x4, x22, x28 +; CHECK-NEXT: adrp x22, :got:var_46 +; CHECK-NEXT: mov x8, xzr +; CHECK-NEXT: ldr x22, [x22, :got_lo12:var_46] +; CHECK-NEXT: str x4, [x22] +; CHECK-NEXT: mov x4, #-18403 // =0xffffffffffffb81d +; CHECK-NEXT: movk x4, #58909, lsl #16 +; CHECK-NEXT: .LBB0_27: // %for.inc371.us +; CHECK-NEXT: // in Loop: Header=BB0_28 Depth=5 +; CHECK-NEXT: mov w22, #-18978 // =0xffffb5de +; CHECK-NEXT: orr x23, x23, #0x1 +; CHECK-NEXT: mov x24, xzr +; CHECK-NEXT: mul w12, w12, w22 +; CHECK-NEXT: mov x22, x5 +; CHECK-NEXT: tbz w0, #0, .LBB0_36 +; CHECK-NEXT: .LBB0_28: // %for.body194.us +; CHECK-NEXT: // Parent Loop BB0_4 Depth=1 +; CHECK-NEXT: // Parent Loop BB0_6 Depth=2 +; CHECK-NEXT: // Parent Loop BB0_8 Depth=3 +; CHECK-NEXT: // Parent Loop BB0_10 Depth=4 +; CHECK-NEXT: // => This Inner Loop Header: Depth=5 +; CHECK-NEXT: cbnz wzr, .LBB0_30 +; CHECK-NEXT: // %bb.29: // %if.then222.us +; CHECK-NEXT: // in Loop: Header=BB0_28 Depth=5 +; CHECK-NEXT: adrp x27, :got:var_32 +; CHECK-NEXT: ldur w8, [x19, #-12] +; CHECK-NEXT: ldr x27, [x27, :got_lo12:var_32] +; CHECK-NEXT: strh w8, [x27] +; CHECK-NEXT: sxtb w8, w25 +; CHECK-NEXT: bic w25, w8, w8, asr #31 +; CHECK-NEXT: b .LBB0_31 +; CHECK-NEXT: .p2align 5, , 16 +; CHECK-NEXT: .LBB0_30: // in Loop: Header=BB0_28 Depth=5 +; CHECK-NEXT: mov w25, wzr +; CHECK-NEXT: .LBB0_31: // %if.end239.us +; CHECK-NEXT: // in Loop: Header=BB0_28 Depth=5 +; CHECK-NEXT: strb w3, [x16] +; CHECK-NEXT: tst w13, #0xff +; CHECK-NEXT: b.eq .LBB0_33 +; CHECK-NEXT: // %bb.32: // %if.then254.us +; CHECK-NEXT: // in Loop: Header=BB0_28 Depth=5 +; CHECK-NEXT: ldrh w8, [x26, x14, lsl #1] +; CHECK-NEXT: adrp x27, :got:var_35 +; CHECK-NEXT: ldr x27, [x27, :got_lo12:var_35] +; CHECK-NEXT: cmp w8, #0 +; CHECK-NEXT: csel x8, xzr, x7, eq +; CHECK-NEXT: str x8, [x27] +; CHECK-NEXT: strh w1, [x17] +; CHECK-NEXT: .LBB0_33: // %if.end282.us +; CHECK-NEXT: // in Loop: Header=BB0_28 Depth=5 +; CHECK-NEXT: orr x27, x24, x4 +; CHECK-NEXT: adrp x8, :got:var_39 +; CHECK-NEXT: str x27, [x18] +; CHECK-NEXT: ldr x8, [x8, :got_lo12:var_39] +; CHECK-NEXT: str x10, [x8] +; CHECK-NEXT: ldrb w8, [x6, x9] +; CHECK-NEXT: str x8, [x18] +; CHECK-NEXT: mov w8, #1 // =0x1 +; CHECK-NEXT: cbnz x2, .LBB0_27 +; CHECK-NEXT: // %bb.34: // %if.then327.us +; CHECK-NEXT: // in Loop: Header=BB0_28 Depth=5 +; CHECK-NEXT: cbz w8, .LBB0_25 +; CHECK-NEXT: // %bb.35: // in Loop: Header=BB0_28 Depth=5 +; CHECK-NEXT: mov w4, wzr +; CHECK-NEXT: b .LBB0_26 +; CHECK-NEXT: .p2align 5, , 16 +; CHECK-NEXT: .LBB0_36: // %for.cond376.preheader.us +; CHECK-NEXT: // in Loop: Header=BB0_10 Depth=4 +; CHECK-NEXT: mov w3, #1152 // =0x480 +; CHECK-NEXT: mov x22, xzr +; CHECK-NEXT: mov w4, wzr +; CHECK-NEXT: mov x24, x27 +; CHECK-NEXT: lsl x23, x14, #1 +; CHECK-NEXT: mov x27, #-1 // =0xffffffffffffffff +; CHECK-NEXT: madd x14, x14, x3, x11 +; CHECK-NEXT: mov w28, w30 +; CHECK-NEXT: mov w3, #-7680 // =0xffffe200 +; CHECK-NEXT: b .LBB0_39 +; CHECK-NEXT: .p2align 5, , 16 +; CHECK-NEXT: .LBB0_37: // %if.then466.us +; CHECK-NEXT: // in Loop: Header=BB0_39 Depth=5 +; CHECK-NEXT: ldr x28, [sp, #152] // 8-byte Folded Reload +; CHECK-NEXT: ldr x3, [sp, #136] // 8-byte Folded Reload +; CHECK-NEXT: sxtb w4, w4 +; CHECK-NEXT: bic w4, w4, w4, asr #31 +; CHECK-NEXT: str x3, [x28] +; CHECK-NEXT: mov w3, #-7680 // =0xffffe200 +; CHECK-NEXT: .LBB0_38: // %for.inc505.us +; CHECK-NEXT: // in Loop: Header=BB0_39 Depth=5 +; CHECK-NEXT: add x22, x22, #1 +; CHECK-NEXT: add x27, x27, #1 +; CHECK-NEXT: mov w28, wzr +; CHECK-NEXT: cmp x27, #0 +; CHECK-NEXT: b.hs .LBB0_9 +; CHECK-NEXT: .LBB0_39: // %for.body380.us +; CHECK-NEXT: // Parent Loop BB0_4 Depth=1 +; CHECK-NEXT: // Parent Loop BB0_6 Depth=2 +; CHECK-NEXT: // Parent Loop BB0_8 Depth=3 +; CHECK-NEXT: // Parent Loop BB0_10 Depth=4 +; CHECK-NEXT: // => This Inner Loop Header: Depth=5 +; CHECK-NEXT: mov w30, w28 +; CHECK-NEXT: ldrh w28, [x23] +; CHECK-NEXT: tst w0, #0x1 +; CHECK-NEXT: strh w28, [x11] +; CHECK-NEXT: csel w28, w21, w3, ne +; CHECK-NEXT: str w28, [x20] +; CHECK-NEXT: cbz x15, .LBB0_38 +; CHECK-NEXT: // %bb.40: // %if.then436.us +; CHECK-NEXT: // in Loop: Header=BB0_39 Depth=5 +; CHECK-NEXT: ldrh w28, [x14] +; CHECK-NEXT: cbnz w28, .LBB0_37 +; CHECK-NEXT: // %bb.41: // in Loop: Header=BB0_39 Depth=5 +; CHECK-NEXT: mov w4, wzr +; CHECK-NEXT: b .LBB0_38 +; CHECK-NEXT: .LBB0_42: // %for.body41 +; CHECK-NEXT: strb wzr, [x4] +; CHECK-NEXT: strb wzr, [x14] +; CHECK-NEXT: .LBB0_43: // %for.cond563.preheader +; CHECK-NEXT: ldp x20, x19, [sp, #224] // 16-byte Folded Reload +; CHECK-NEXT: ldp x22, x21, [sp, #208] // 16-byte Folded Reload +; CHECK-NEXT: ldp x24, x23, [sp, #192] // 16-byte Folded Reload +; CHECK-NEXT: ldp x26, x25, [sp, #176] // 16-byte Folded Reload +; CHECK-NEXT: ldp x28, x27, [sp, #160] // 16-byte Folded Reload +; CHECK-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload +; CHECK-NEXT: add sp, sp, #240 +; CHECK-NEXT: ret +entry: + br i1 %var_5, label %for.body41.lr.ph, label %for.cond563.preheader + +for.body41.lr.ph: ; preds = %entry + %arrayidx147 = getelementptr i8, ptr %arr_3, i64 120 + %tobool326.not = icmp eq i64 %var_2, 0 + %not353 = xor i64 0, -1 + %add538 = select i1 %var_0, i16 0, i16 1 + br i1 %var_0, label %for.body41.us, label %for.body41 + +for.body41.us: ; preds = %for.cond.cleanup93.us, %for.body41.lr.ph + %var_24.promoted9271009.us = phi i64 [ 0, %for.body41.lr.ph ], [ %6, %for.cond.cleanup93.us ] + %var_37.promoted9301008.us = phi i64 [ 1, %for.body41.lr.ph ], [ 0, %for.cond.cleanup93.us ] + %2 = phi i8 [ 0, %for.body41.lr.ph ], [ 1, %for.cond.cleanup93.us ] + %add4139751001.us = phi i16 [ 0, %for.body41.lr.ph ], [ 1, %for.cond.cleanup93.us ] + %3 = phi i8 [ 0, %for.body41.lr.ph ], [ %var_10, %for.cond.cleanup93.us ] + store i32 %var_6, ptr %arr_3, align 4 + store i8 %var_10, ptr %arr_3, align 1 + br label %for.body67.us + +for.body67.us: ; preds = %for.cond.cleanup93.us, %for.body41.us + %4 = phi i8 [ %3, %for.body41.us ], [ 0, %for.cond.cleanup93.us ] + %add413977.us = phi i16 [ %add4139751001.us, %for.body41.us ], [ %add413.us17, %for.cond.cleanup93.us ] + %5 = phi i8 [ %2, %for.body41.us ], [ %.sroa.speculated829.us, %for.cond.cleanup93.us ] + %conv64922.us = phi i32 [ 1, %for.body41.us ], [ 0, %for.cond.cleanup93.us ] + %6 = phi i64 [ %var_24.promoted9271009.us, %for.body41.us ], [ %.sroa.speculated832.us, %for.cond.cleanup93.us ] + %mul354903918.us = phi i64 [ %var_37.promoted9301008.us, %for.body41.us ], [ 0, %for.cond.cleanup93.us ] + %i_2.0921.us = zext i32 %var_15 to i64 + %.sroa.speculated832.us = tail call i64 @llvm.umin.i64(i64 %var_24.promoted9271009.us, i64 -30) + %sext1023 = shl i64 %i_2.0921.us, 1 + %idxprom138.us162 = ashr i64 %sext1023, 1 + %gep889.us = getelementptr [24 x i16], ptr %arr_19, i64 %idxprom138.us16 + %arrayidx149.us = getelementptr i8, ptr %arrayidx147, i64 %idxprom138.us162 + %arrayidx319.us = getelementptr [24 x i8], ptr null, i64 %idxprom138.us162 + %7 = sext i32 %conv64922.us to i64 + %8 = getelementptr i32, ptr %arr_12, i64 %7 + %arrayidx226.us = getelementptr i8, ptr %8, i64 -12 + br label %for.cond95.preheader.us + +for.cond.cleanup93.us: ; preds = %for.cond.cleanup98.us + br i1 %var_5, label %for.body67.us, label %for.body41.us + +for.cond.cleanup98.us: ; preds = %for.cond510.preheader.us + br i1 %var_0, label %for.cond.cleanup93.us, label %for.cond95.preheader.us + +for.body99.us: ; preds = %for.cond95.preheader.us, %for.cond510.preheader.us + %mul287985.us = phi i16 [ 0, %for.cond95.preheader.us ], [ %mul287.us, %for.cond510.preheader.us ] + %9 = phi i8 [ %29, %for.cond95.preheader.us ], [ %var_14, %for.cond510.preheader.us ] + %add413979.us = phi i16 [ %add413978.us, %for.cond95.preheader.us ], [ %add413.us17, %for.cond510.preheader.us ] + %10 = phi i32 [ 0, %for.cond95.preheader.us ], [ %26, %for.cond510.preheader.us ] + %mul354905.us = phi i64 [ %mul354904.us, %for.cond95.preheader.us ], [ %mul354907.us, %for.cond510.preheader.us ] + %sub283896.us = phi i64 [ 1, %for.cond95.preheader.us ], [ %sub283.us, %for.cond510.preheader.us ] + %conv96880.us = phi i64 [ 1, %for.cond95.preheader.us ], [ 0, %for.cond510.preheader.us ] + %.sroa.speculated829.us = tail call i8 @llvm.smin.i8(i8 %30, i8 0) + br label %for.body113.us + +for.body380.us: ; preds = %for.cond376.preheader.us, %for.inc505.us + %indvars.iv1018 = phi i64 [ 0, %for.cond376.preheader.us ], [ %indvars.iv.next1019, %for.inc505.us ] + %11 = phi i8 [ 0, %for.cond376.preheader.us ], [ %13, %for.inc505.us ] + %add413980.us = phi i16 [ %add413979.us, %for.cond376.preheader.us ], [ 0, %for.inc505.us ] + %12 = load i16, ptr %arrayidx384.us, align 2 + store i16 %12, ptr %invariant.gep875.us, align 2 + %add413.us17 = or i16 %add413980.us, 0 + %arrayidx416.us = getelementptr i16, ptr %arr_13, i64 %indvars.iv1018 + %conv419.us = select i1 %var_0, i32 36006, i32 -7680 + store i32 %conv419.us, ptr @var_50, align 4 + %tobool435.not.us = icmp eq i64 %mul, 0 + br i1 %tobool435.not.us, label %for.inc505.us, label %if.then436.us + +if.then436.us: ; preds = %for.body380.us + %.sroa.speculated817.us = tail call i8 @llvm.smax.i8(i8 %11, i8 0) + %cond464.in.us = load i16, ptr %gep876.us, align 2 + %tobool465.not.us = icmp eq i16 %cond464.in.us, 0 + br i1 %tobool465.not.us, label %for.inc505.us, label %if.then466.us + +if.then466.us: ; preds = %if.then436.us + store i64 %conv35, ptr %arr_3, align 8 + br label %for.inc505.us + +for.inc505.us: ; preds = %if.then466.us, %if.then436.us, %for.body380.us + %13 = phi i8 [ %11, %for.body380.us ], [ %.sroa.speculated817.us, %if.then466.us ], [ 0, %if.then436.us ] + %indvars.iv.next1019 = add i64 %indvars.iv1018, 1 + %cmp378.us = icmp ult i64 %indvars.iv1018, 0 + br i1 %cmp378.us, label %for.body380.us, label %for.cond510.preheader.us + +for.body194.us: ; preds = %if.end.us.7, %for.inc371.us + %indvars.iv = phi i64 [ 0, %if.end.us.7 ], [ %indvars.iv.next, %for.inc371.us ] + %mul287986.us = phi i16 [ %mul287985.us, %if.end.us.7 ], [ %mul287.us, %for.inc371.us ] + %14 = phi i8 [ %9, %if.end.us.7 ], [ %16, %for.inc371.us ] + %mul354906.us = phi i64 [ %mul354905.us, %if.end.us.7 ], [ %var_11, %for.inc371.us ] + %sub283897.us = phi i64 [ %sub283896.us, %if.end.us.7 ], [ 0, %for.inc371.us ] + %tobool221.not.us = icmp eq i32 1, 0 + br i1 %tobool221.not.us, label %if.end239.us, label %if.then222.us + +if.then222.us: ; preds = %for.body194.us + %15 = load i32, ptr %arrayidx226.us, align 4 + %conv227.us = trunc i32 %15 to i16 + store i16 %conv227.us, ptr @var_32, align 2 + %.sroa.speculated820.us = tail call i8 @llvm.smax.i8(i8 %14, i8 0) + br label %if.end239.us + +if.end239.us: ; preds = %if.then222.us, %for.body194.us + %16 = phi i8 [ %.sroa.speculated820.us, %if.then222.us ], [ 0, %for.body194.us ] + store i8 -107, ptr %arr_7, align 1 + %tobool253.not.us = icmp eq i8 %0, 0 + br i1 %tobool253.not.us, label %if.end282.us, label %if.then254.us + +if.then254.us: ; preds = %if.end239.us + %17 = load i16, ptr %arrayidx259.us, align 2 + %tobool261.not.us = icmp eq i16 %17, 0 + %conv268.us = select i1 %tobool261.not.us, i64 0, i64 %var_16 + store i64 %conv268.us, ptr @var_35, align 8 + %gep867.us = getelementptr [24 x [24 x i64]], ptr null, i64 %indvars.iv + store i16 %var_1, ptr %arr_6, align 2 + br label %if.end282.us + +if.end282.us: ; preds = %if.then254.us, %if.end239.us + %sub283.us = or i64 %sub283897.us, -434259939 + store i64 %sub283.us, ptr %arr_4, align 8 + %mul287.us = mul i16 %mul287986.us, -18978 + store i64 0, ptr @var_39, align 8 + %18 = load i8, ptr %arrayidx321.us, align 1 + %conv322.us = zext i8 %18 to i64 + store i64 %conv322.us, ptr %arr_4, align 8 + br i1 %tobool326.not, label %if.then327.us, label %for.inc371.us + +if.then327.us: ; preds = %if.end282.us + %tobool330.not.us = icmp eq i32 0, 0 + br i1 %tobool330.not.us, label %cond.end345.us, label %cond.true331.us + +cond.true331.us: ; preds = %if.then327.us + %19 = load i8, ptr null, align 1 + %20 = sext i8 %19 to i16 + br label %cond.end345.us + +cond.end345.us: ; preds = %cond.true331.us, %if.then327.us + %cond346.us = phi i16 [ %20, %cond.true331.us ], [ 0, %if.then327.us ] + store i16 %cond346.us, ptr %arr_4, align 2 + %mul354.us = mul i64 %mul354906.us, %not353 + store i64 %mul354.us, ptr @var_46, align 8 + br label %for.inc371.us + +for.inc371.us: ; preds = %cond.end345.us, %if.end282.us + %mul354907.us = phi i64 [ 1, %if.end282.us ], [ 0, %cond.end345.us ] + %indvars.iv.next = or i64 %indvars.iv, 1 + br i1 %var_0, label %for.body194.us, label %for.cond376.preheader.us + +cond.true146.us: ; preds = %for.cond131.preheader.us + %21 = load i8, ptr %arrayidx149.us, align 1 + %conv150.us = sext i8 %21 to i32 + br label %cond.end154.us + +cond.end154.us: ; preds = %for.cond131.preheader.us, %cond.true146.us + %cond155.us = phi i32 [ %conv150.us, %cond.true146.us ], [ 0, %for.cond131.preheader.us ] + %shl.us = shl i32 %div.us, %cond155.us + %tobool157.not.us = icmp eq i32 %shl.us, 0 + br i1 %tobool157.not.us, label %if.end.us, label %if.then.us + +if.then.us: ; preds = %cond.end154.us + store i32 0, ptr %arr_4, align 4 + br label %if.end.us + +if.end.us: ; preds = %if.then.us, %cond.end154.us + %22 = phi i32 [ 0, %if.then.us ], [ %10, %cond.end154.us ] + store i8 %1, ptr %arr_4, align 1 + call void @llvm.assume(i1 true) + %23 = load i8, ptr %arrayidx149.us, align 1 + %conv150.us.2 = sext i8 %23 to i32 + %shl.us.2 = shl i32 18984, %conv150.us.2 + %tobool157.not.us.2 = icmp eq i32 %shl.us.2, 0 + br i1 %tobool157.not.us.2, label %if.then.us.5, label %if.then.us.2 + +if.then.us.2: ; preds = %if.end.us + %.sroa.speculated826.us.2 = tail call i32 @llvm.smin.i32(i32 %10, i32 0) + store i8 0, ptr %arr_4, align 1 + br label %if.then.us.5 + +if.then.us.5: ; preds = %if.then.us.2, %if.end.us + %24 = phi i32 [ 0, %if.then.us.2 ], [ %22, %if.end.us ] + %.sroa.speculated826.us.5 = tail call i32 @llvm.smin.i32(i32 %24, i32 1410036665) + br i1 %var_0, label %cond.end154.us.7, label %cond.true146.us.7 + +cond.true146.us.7: ; preds = %if.then.us.5 + %25 = load i8, ptr %arrayidx149.us, align 1 + %conv150.us.7 = sext i8 %25 to i32 + br label %cond.end154.us.7 + +cond.end154.us.7: ; preds = %cond.true146.us.7, %if.then.us.5 + %cond155.us.7 = phi i32 [ %conv150.us.7, %cond.true146.us.7 ], [ 0, %if.then.us.5 ] + %shl.us.7 = shl i32 18984, %cond155.us.7 + %tobool157.not.us.7 = icmp eq i32 %shl.us.7, 0 + br i1 %tobool157.not.us.7, label %if.end.us.7, label %if.then.us.7 + +if.then.us.7: ; preds = %cond.end154.us.7 + store i32 0, ptr %arr_3, align 4 + br label %if.end.us.7 + +if.end.us.7: ; preds = %if.then.us.7, %cond.end154.us.7 + %26 = phi i32 [ 0, %if.then.us.7 ], [ %.sroa.speculated826.us.5, %cond.end154.us.7 ] + %arrayidx259.us = getelementptr i16, ptr %arrayidx257.us, i64 %conv96880.us + br label %for.body194.us + +for.body113.us: ; preds = %for.body113.us, %for.body99.us + br i1 %var_0, label %for.body113.us, label %for.cond131.preheader.us + +for.cond510.preheader.us: ; preds = %for.inc505.us + %cmp97.us = icmp slt i16 %add538, 0 + br i1 %cmp97.us, label %for.body99.us, label %for.cond.cleanup98.us + +for.cond376.preheader.us: ; preds = %for.inc371.us + %arrayidx384.us = getelementptr i16, ptr null, i64 %conv96880.us + %gep876.us = getelementptr [24 x [24 x i16]], ptr %invariant.gep875.us, i64 %conv96880.us + br label %for.body380.us + +for.cond131.preheader.us: ; preds = %for.body113.us + store i8 %var_3, ptr %arr_4, align 1 + %27 = load i16, ptr %gep884.us, align 2 + %28 = mul i16 18984, %27 + %div.us = zext i16 %28 to i32 + %tobool145.not.us = icmp eq i8 0, 0 + br i1 %tobool145.not.us, label %cond.end154.us, label %cond.true146.us + +for.cond95.preheader.us: ; preds = %for.cond.cleanup98.us, %for.body67.us + %indvars.iv1021 = phi i64 [ 1, %for.cond.cleanup98.us ], [ 0, %for.body67.us ] + %29 = phi i8 [ %16, %for.cond.cleanup98.us ], [ %4, %for.body67.us ] + %add413978.us = phi i16 [ %var_4, %for.cond.cleanup98.us ], [ %add413977.us, %for.body67.us ] + %30 = phi i8 [ %.sroa.speculated829.us, %for.cond.cleanup98.us ], [ %5, %for.body67.us ] + %mul354904.us = phi i64 [ 0, %for.cond.cleanup98.us ], [ %mul354903918.us, %for.body67.us ] + %gep884.us = getelementptr [24 x [24 x i16]], ptr %gep889.us, i64 %indvars.iv1021 + %arrayidx321.us = getelementptr i8, ptr %arrayidx319.us, i64 %indvars.iv1021 + %arrayidx257.us = getelementptr [24 x i16], ptr null, i64 %indvars.iv1021 + br label %for.body99.us + +for.cond563.preheader: ; preds = %for.body41, %entry + ret void + +for.body41: ; preds = %for.body41.lr.ph + store i8 0, ptr %arr_12, align 1 + store i8 0, ptr %arr_3, align 1 + br label %for.cond563.preheader +} + +attributes #0 = { nounwind "frame-pointer"="non-leaf" "target-cpu"="grace" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) } diff --git a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll index 57a1e4c..ec92edb 100644 --- a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll +++ b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll @@ -3385,7 +3385,7 @@ declare half @llvm.canonicalize.f16(half) declare <2 x half> @llvm.canonicalize.v2f16(<2 x half>) attributes #0 = { nounwind "amdgpu-ieee"="false" } -attributes #1 = { nounwind "unsafe-fp-math"="true" "no-nans-fp-math"="true" } +attributes #1 = { nounwind "no-nans-fp-math"="true" } ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; GFX11NONANS-FAKE16: {{.*}} ; GFX11NONANS-TRUE16: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.f64.ll b/llvm/test/CodeGen/AMDGPU/fdiv.f64.ll index acb32d4..11476a6 100644 --- a/llvm/test/CodeGen/AMDGPU/fdiv.f64.ll +++ b/llvm/test/CodeGen/AMDGPU/fdiv.f64.ll @@ -127,7 +127,7 @@ define amdgpu_kernel void @s_fdiv_v4f64(ptr addrspace(1) %out, <4 x double> %num ; GCN-LABEL: {{^}}div_fast_2_x_pat_f64: ; GCN: v_mul_f64 [[MUL:v\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}, 0.5 ; GCN: buffer_store_dwordx2 [[MUL]] -define amdgpu_kernel void @div_fast_2_x_pat_f64(ptr addrspace(1) %out) #1 { +define amdgpu_kernel void @div_fast_2_x_pat_f64(ptr addrspace(1) %out) #0 { %x = load double, ptr addrspace(1) poison %rcp = fdiv fast double %x, 2.0 store double %rcp, ptr addrspace(1) %out, align 4 @@ -139,7 +139,7 @@ define amdgpu_kernel void @div_fast_2_x_pat_f64(ptr addrspace(1) %out) #1 { ; GCN-DAG: v_mov_b32_e32 v[[K_HI:[0-9]+]], 0x3fb99999 ; GCN: v_mul_f64 [[MUL:v\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}, v[[[K_LO]]:[[K_HI]]] ; GCN: buffer_store_dwordx2 [[MUL]] -define amdgpu_kernel void @div_fast_k_x_pat_f64(ptr addrspace(1) %out) #1 { +define amdgpu_kernel void @div_fast_k_x_pat_f64(ptr addrspace(1) %out) #0 { %x = load double, ptr addrspace(1) poison %rcp = fdiv fast double %x, 10.0 store double %rcp, ptr addrspace(1) %out, align 4 @@ -151,7 +151,7 @@ define amdgpu_kernel void @div_fast_k_x_pat_f64(ptr addrspace(1) %out) #1 { ; GCN-DAG: v_mov_b32_e32 v[[K_HI:[0-9]+]], 0xbfb99999 ; GCN: v_mul_f64 [[MUL:v\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}, v[[[K_LO]]:[[K_HI]]] ; GCN: buffer_store_dwordx2 [[MUL]] -define amdgpu_kernel void @div_fast_neg_k_x_pat_f64(ptr addrspace(1) %out) #1 { +define amdgpu_kernel void @div_fast_neg_k_x_pat_f64(ptr addrspace(1) %out) #0 { %x = load double, ptr addrspace(1) poison %rcp = fdiv fast double %x, -10.0 store double %rcp, ptr addrspace(1) %out, align 4 @@ -159,4 +159,3 @@ define amdgpu_kernel void @div_fast_neg_k_x_pat_f64(ptr addrspace(1) %out) #1 { } attributes #0 = { nounwind } -attributes #1 = { nounwind "unsafe-fp-math"="true" } diff --git a/llvm/test/CodeGen/AMDGPU/fmad-formation-fmul-distribute-denormal-mode.ll b/llvm/test/CodeGen/AMDGPU/fmad-formation-fmul-distribute-denormal-mode.ll index 92eb4a6..0a266bc 100644 --- a/llvm/test/CodeGen/AMDGPU/fmad-formation-fmul-distribute-denormal-mode.ll +++ b/llvm/test/CodeGen/AMDGPU/fmad-formation-fmul-distribute-denormal-mode.ll @@ -284,4 +284,4 @@ define <2 x float> @unsafe_fast_fmul_fsub_ditribute_post_legalize(float %arg0, < ret <2 x float> %tmp1 } -attributes #0 = { "no-infs-fp-math"="true" "unsafe-fp-math"="true" } +attributes #0 = { "no-infs-fp-math"="true" } diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll index bc85dc2..3e513de 100644 --- a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll @@ -219,8 +219,8 @@ define <2 x bfloat> @v_test_fmed3_r_i_i_v2bf16_minimumnum_maximumnum(<2 x bfloat } attributes #0 = { nounwind readnone } -attributes #1 = { nounwind "unsafe-fp-math"="false" "no-nans-fp-math"="false" } -attributes #2 = { nounwind "unsafe-fp-math"="false" "no-nans-fp-math"="true" } +attributes #1 = { nounwind "no-nans-fp-math"="false" } +attributes #2 = { nounwind "no-nans-fp-math"="true" } ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; GFX11: {{.*}} ; GFX11-SDAG: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.ll b/llvm/test/CodeGen/AMDGPU/fmed3.ll index 3145a27..60ac0b9 100644 --- a/llvm/test/CodeGen/AMDGPU/fmed3.ll +++ b/llvm/test/CodeGen/AMDGPU/fmed3.ll @@ -8905,4 +8905,4 @@ declare half @llvm.minnum.f16(half, half) #0 declare half @llvm.maxnum.f16(half, half) #0 attributes #0 = { nounwind readnone } -attributes #2 = { nounwind "unsafe-fp-math"="false" "no-nans-fp-math"="true" } +attributes #2 = { nounwind "no-nans-fp-math"="true" } diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll index d8bbda1..69d1ee3f 100644 --- a/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll @@ -159,7 +159,7 @@ declare half @llvm.amdgcn.interp.p2.f16(float, float, i32, i32, i1, i32) #0 attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" } attributes #1 = { nounwind readnone } -attributes #2 = { nounwind "unsafe-fp-math"="true" } +attributes #2 = { nounwind } attributes #3 = { nounwind "no-signed-zeros-fp-math"="true" } attributes #4 = { nounwind "amdgpu-ieee"="false" "denormal-fp-math-f32"="preserve-sign,preserve-sign" } ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll index aaea4f7..b3202cb 100644 --- a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll +++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll @@ -8006,7 +8006,7 @@ declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #0 attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" } attributes #1 = { nounwind readnone } -attributes #2 = { nounwind "unsafe-fp-math"="true" } +attributes #2 = { nounwind } attributes #3 = { nounwind "no-signed-zeros-fp-math"="true" } ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; GCN-NSZ: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll index 6f91222..d8cbdb1 100644 --- a/llvm/test/CodeGen/AMDGPU/frem.ll +++ b/llvm/test/CodeGen/AMDGPU/frem.ll @@ -2048,7 +2048,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; GFX1200-FAKE16-NEXT: v_fmac_f16_e32 v1, v3, v2 ; GFX1200-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1] ; GFX1200-FAKE16-NEXT: s_endpgm - ptr addrspace(1) %in2) #1 { + ptr addrspace(1) %in2) #0 { %gep2 = getelementptr half, ptr addrspace(1) %in2, i32 4 %r0 = load half, ptr addrspace(1) %in1, align 4 %r1 = load half, ptr addrspace(1) %gep2, align 4 @@ -3417,7 +3417,7 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; GFX1200-NEXT: v_fmac_f32_e32 v1, v3, v2 ; GFX1200-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX1200-NEXT: s_endpgm - ptr addrspace(1) %in2) #1 { + ptr addrspace(1) %in2) #0 { %gep2 = getelementptr float, ptr addrspace(1) %in2, i32 4 %r0 = load float, ptr addrspace(1) %in1, align 4 %r1 = load float, ptr addrspace(1) %gep2, align 4 @@ -4821,7 +4821,7 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; GFX1200-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] ; GFX1200-NEXT: global_store_b64 v12, v[0:1], s[0:1] ; GFX1200-NEXT: s_endpgm - ptr addrspace(1) %in2) #1 { + ptr addrspace(1) %in2) #0 { %r0 = load double, ptr addrspace(1) %in1, align 8 %r1 = load double, ptr addrspace(1) %in2, align 8 %r2 = frem afn double %r0, %r1 @@ -18918,7 +18918,4 @@ define amdgpu_kernel void @frem_v2f64_const(ptr addrspace(1) %out) #0 { -attributes #0 = { nounwind "unsafe-fp-math"="false" "denormal-fp-math-f32"="preserve-sign,preserve-sign" } -attributes #1 = { nounwind "unsafe-fp-math"="true" "denormal-fp-math-f32"="preserve-sign,preserve-sign" } - - +attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" } diff --git a/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll b/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll index 1b74ddf..9b97981 100644 --- a/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll +++ b/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll @@ -2870,7 +2870,7 @@ define double @v_sqrt_f64__enough_unsafe_attrs(double %x) #3 { ret double %result } -define double @v_sqrt_f64__unsafe_attr(double %x) #4 { +define double @v_sqrt_f64__unsafe_attr(double %x) { ; GFX6-SDAG-LABEL: v_sqrt_f64__unsafe_attr: ; GFX6-SDAG: ; %bb.0: ; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3449,7 +3449,6 @@ declare i32 @llvm.amdgcn.readfirstlane(i32) #1 attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } attributes #1 = { convergent nounwind willreturn memory(none) } attributes #3 = { "no-nans-fp-math"="true" "no-infs-fp-math"="true" } -attributes #4 = { "unsafe-fp-math"="true" } ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; GFX6: {{.*}} ; GFX8: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/fsqrt.r600.ll b/llvm/test/CodeGen/AMDGPU/fsqrt.r600.ll index 9f19bcb..c93c077 100644 --- a/llvm/test/CodeGen/AMDGPU/fsqrt.r600.ll +++ b/llvm/test/CodeGen/AMDGPU/fsqrt.r600.ll @@ -239,4 +239,4 @@ declare <2 x float> @llvm.sqrt.v2f32(<2 x float> %in) #0 declare <4 x float> @llvm.sqrt.v4f32(<4 x float> %in) #0 attributes #0 = { nounwind readnone } -attributes #1 = { nounwind "unsafe-fp-math"="true" } +attributes #1 = { nounwind } diff --git a/llvm/test/CodeGen/AMDGPU/inline-attr.ll b/llvm/test/CodeGen/AMDGPU/inline-attr.ll index 4e93eca..c33b3344 100644 --- a/llvm/test/CodeGen/AMDGPU/inline-attr.ll +++ b/llvm/test/CodeGen/AMDGPU/inline-attr.ll @@ -36,18 +36,18 @@ entry: ret void } -attributes #0 = { nounwind "uniform-work-group-size"="false" "unsafe-fp-math"="true"} -attributes #1 = { nounwind "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" } +attributes #0 = { nounwind "uniform-work-group-size"="false"} +attributes #1 = { nounwind "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" } ;. -; UNSAFE: attributes #[[ATTR0]] = { nounwind "uniform-work-group-size"="false" "unsafe-fp-math"="true" } -; UNSAFE: attributes #[[ATTR1]] = { nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "uniform-work-group-size"="false" "unsafe-fp-math"="true" } +; UNSAFE: attributes #[[ATTR0]] = { nounwind "uniform-work-group-size"="false" } +; UNSAFE: attributes #[[ATTR1]] = { nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "uniform-work-group-size"="false" } ;. -; NONANS: attributes #[[ATTR0]] = { nounwind "no-nans-fp-math"="true" "uniform-work-group-size"="false" "unsafe-fp-math"="true" } -; NONANS: attributes #[[ATTR1]] = { nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="true" "uniform-work-group-size"="false" "unsafe-fp-math"="true" } +; NONANS: attributes #[[ATTR0]] = { nounwind "no-nans-fp-math"="true" "uniform-work-group-size"="false" } +; NONANS: attributes #[[ATTR1]] = { nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="true" "uniform-work-group-size"="false" } ;. -; NOINFS: attributes #[[ATTR0]] = { nounwind "no-infs-fp-math"="true" "uniform-work-group-size"="false" "unsafe-fp-math"="true" } -; NOINFS: attributes #[[ATTR1]] = { nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="true" "no-nans-fp-math"="false" "uniform-work-group-size"="false" "unsafe-fp-math"="true" } +; NOINFS: attributes #[[ATTR0]] = { nounwind "no-infs-fp-math"="true" "uniform-work-group-size"="false" } +; NOINFS: attributes #[[ATTR1]] = { nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="true" "no-nans-fp-math"="false" "uniform-work-group-size"="false" } ;. ; UNSAFE: [[META0]] = !{} ;. diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.add.min.max.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.add.min.max.ll new file mode 100644 index 0000000..99421d4 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.add.min.max.ll @@ -0,0 +1,191 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GFX1250-SDAG %s +; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GFX1250-GISEL %s + +declare i32 @llvm.amdgcn.add.min.i32(i32, i32, i32, i1) +declare i32 @llvm.amdgcn.add.max.i32(i32, i32, i32, i1) +declare i32 @llvm.amdgcn.add.min.u32(i32, i32, i32, i1) +declare i32 @llvm.amdgcn.add.max.u32(i32, i32, i32, i1) +declare <2 x i16> @llvm.amdgcn.pk.add.min.i16(<2 x i16>, <2 x i16>, <2 x i16>, i1) +declare <2 x i16> @llvm.amdgcn.pk.add.max.i16(<2 x i16>, <2 x i16>, <2 x i16>, i1) +declare <2 x i16> @llvm.amdgcn.pk.add.min.u16(<2 x i16>, <2 x i16>, <2 x i16>, i1) +declare <2 x i16> @llvm.amdgcn.pk.add.max.u16(<2 x i16>, <2 x i16>, <2 x i16>, i1) + +define i32 @test_add_min_i32_vvv(i32 %a, i32 %b, i32 %c) { +; GCN-LABEL: test_add_min_i32_vvv: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_add_min_i32 v0, v0, v1, v2 +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call i32 @llvm.amdgcn.add.min.i32(i32 %a, i32 %b, i32 %c, i1 0) + ret i32 %ret +} + +define i32 @test_add_min_i32_ssi_clamp(i32 inreg %a, i32 inreg %b) { +; GCN-LABEL: test_add_min_i32_ssi_clamp: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_add_min_i32 v0, s0, s1, 1 clamp +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call i32 @llvm.amdgcn.add.min.i32(i32 %a, i32 %b, i32 1, i1 1) + ret i32 %ret +} + +define i32 @test_add_min_u32_vvv(i32 %a, i32 %b, i32 %c) { +; GCN-LABEL: test_add_min_u32_vvv: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_add_min_u32 v0, v0, v1, v2 +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call i32 @llvm.amdgcn.add.min.u32(i32 %a, i32 %b, i32 %c, i1 0) + ret i32 %ret +} + +define i32 @test_add_min_u32_ssi_clamp(i32 inreg %a, i32 inreg %b) { +; GCN-LABEL: test_add_min_u32_ssi_clamp: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_add_min_u32 v0, s0, s1, 1 clamp +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call i32 @llvm.amdgcn.add.min.u32(i32 %a, i32 %b, i32 1, i1 1) + ret i32 %ret +} + +define i32 @test_add_max_i32_vvv(i32 %a, i32 %b, i32 %c) { +; GCN-LABEL: test_add_max_i32_vvv: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_add_max_i32 v0, v0, v1, v2 +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call i32 @llvm.amdgcn.add.max.i32(i32 %a, i32 %b, i32 %c, i1 0) + ret i32 %ret +} + +define i32 @test_add_max_i32_ssi_clamp(i32 inreg %a, i32 inreg %b) { +; GCN-LABEL: test_add_max_i32_ssi_clamp: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_add_max_i32 v0, s0, s1, 1 clamp +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call i32 @llvm.amdgcn.add.max.i32(i32 %a, i32 %b, i32 1, i1 1) + ret i32 %ret +} + +define i32 @test_add_max_u32_vvv(i32 %a, i32 %b, i32 %c) { +; GCN-LABEL: test_add_max_u32_vvv: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_add_max_u32 v0, v0, v1, v2 +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call i32 @llvm.amdgcn.add.max.u32(i32 %a, i32 %b, i32 %c, i1 0) + ret i32 %ret +} + +define i32 @test_add_max_u32_ssi_clamp(i32 inreg %a, i32 inreg %b) { +; GCN-LABEL: test_add_max_u32_ssi_clamp: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_add_max_u32 v0, s0, s1, 1 clamp +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call i32 @llvm.amdgcn.add.max.u32(i32 %a, i32 %b, i32 1, i1 1) + ret i32 %ret +} + +define <2 x i16> @test_add_min_i16_vvv(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c) { +; GCN-LABEL: test_add_min_i16_vvv: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_pk_add_min_i16 v0, v0, v1, v2 +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call <2 x i16> @llvm.amdgcn.pk.add.min.i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, i1 0) + ret <2 x i16> %ret +} + +define <2 x i16> @test_add_min_i16_ssi_clamp(<2 x i16> inreg %a, <2 x i16> inreg %b) { +; GCN-LABEL: test_add_min_i16_ssi_clamp: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_pk_add_min_i16 v0, s0, s1, 1 op_sel_hi:[1,1,0] clamp +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call <2 x i16> @llvm.amdgcn.pk.add.min.i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> <i16 1, i16 1>, i1 1) + ret <2 x i16> %ret +} + +define <2 x i16> @test_add_min_u16_vvv(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c) { +; GCN-LABEL: test_add_min_u16_vvv: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_pk_add_min_u16 v0, v0, v1, v2 +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call <2 x i16> @llvm.amdgcn.pk.add.min.u16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, i1 0) + ret <2 x i16> %ret +} + +define <2 x i16> @test_add_min_u16_ssi_clamp(<2 x i16> inreg %a, <2 x i16> inreg %b) { +; GCN-LABEL: test_add_min_u16_ssi_clamp: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_pk_add_min_u16 v0, s0, s1, 1 op_sel_hi:[1,1,0] clamp +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call <2 x i16> @llvm.amdgcn.pk.add.min.u16(<2 x i16> %a, <2 x i16> %b, <2 x i16> <i16 1, i16 1>, i1 1) + ret <2 x i16> %ret +} + +define <2 x i16> @test_add_max_i16_vvv(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c) { +; GCN-LABEL: test_add_max_i16_vvv: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_pk_add_max_i16 v0, v0, v1, v2 +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call <2 x i16> @llvm.amdgcn.pk.add.max.i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, i1 0) + ret <2 x i16> %ret +} + +define <2 x i16> @test_add_max_i16_ssi_clamp(<2 x i16> inreg %a, <2 x i16> inreg %b) { +; GCN-LABEL: test_add_max_i16_ssi_clamp: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_pk_add_max_i16 v0, s0, s1, 1 op_sel_hi:[1,1,0] clamp +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call <2 x i16> @llvm.amdgcn.pk.add.max.i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> <i16 1, i16 1>, i1 1) + ret <2 x i16> %ret +} + +define <2 x i16> @test_add_max_u16_vvv(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c) { +; GCN-LABEL: test_add_max_u16_vvv: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_pk_add_max_u16 v0, v0, v1, v2 +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call <2 x i16> @llvm.amdgcn.pk.add.max.u16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, i1 0) + ret <2 x i16> %ret +} + +define <2 x i16> @test_add_max_u16_ssi_clamp(<2 x i16> inreg %a, <2 x i16> inreg %b) { +; GCN-LABEL: test_add_max_u16_ssi_clamp: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: v_pk_add_max_u16 v0, s0, s1, 1 op_sel_hi:[1,1,0] clamp +; GCN-NEXT: s_set_pc_i64 s[30:31] + %ret = tail call <2 x i16> @llvm.amdgcn.pk.add.max.u16(<2 x i16> %a, <2 x i16> %b, <2 x i16> <i16 1, i16 1>, i1 1) + ret <2 x i16> %ret +} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GFX1250-GISEL: {{.*}} +; GFX1250-SDAG: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll index 883db20..e30a586 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll @@ -1485,7 +1485,7 @@ define float @v_exp2_f32_fast(float %in) { ret float %result } -define float @v_exp2_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" { +define float @v_exp2_f32_unsafe_math_attr(float %in) { ; SI-SDAG-LABEL: v_exp2_f32_unsafe_math_attr: ; SI-SDAG: ; %bb.0: ; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll index 0854134..61a777f 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll @@ -1907,7 +1907,7 @@ define float @v_log2_f32_fast(float %in) { ret float %result } -define float @v_log2_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" { +define float @v_log2_f32_unsafe_math_attr(float %in) { ; SI-SDAG-LABEL: v_log2_f32_unsafe_math_attr: ; SI-SDAG: ; %bb.0: ; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) diff --git a/llvm/test/CodeGen/AMDGPU/minmax.ll b/llvm/test/CodeGen/AMDGPU/minmax.ll index d578d2e..60570bd 100644 --- a/llvm/test/CodeGen/AMDGPU/minmax.ll +++ b/llvm/test/CodeGen/AMDGPU/minmax.ll @@ -1296,4 +1296,4 @@ declare half @llvm.minnum.f16(half, half) declare half @llvm.maxnum.f16(half, half) declare float @llvm.minnum.f32(float, float) declare float @llvm.maxnum.f32(float, float) -attributes #0 = { nounwind "unsafe-fp-math"="false" "no-nans-fp-math"="true" } +attributes #0 = { nounwind "no-nans-fp-math"="true" } diff --git a/llvm/test/CodeGen/AMDGPU/stackguard.ll b/llvm/test/CodeGen/AMDGPU/stackguard.ll new file mode 100644 index 0000000..393686f --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/stackguard.ll @@ -0,0 +1,14 @@ +; RUN: not llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=null %s 2>&1 | FileCheck %s +; RUN: not llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=null %s 2>&1 | FileCheck %s + +; FIXME: To actually support stackguard, need to fix intrinsic to +; return pointer in any address space. + +; CHECK: error: unable to lower stackguard +define i1 @test_stackguard(ptr %p1) { + %p2 = call ptr @llvm.stackguard() + %res = icmp ne ptr %p2, %p1 + ret i1 %res +} + +declare ptr @llvm.stackguard() diff --git a/llvm/test/CodeGen/ARM/call-graph-section-addrtaken.ll b/llvm/test/CodeGen/ARM/call-graph-section-addrtaken.ll index 972a470..cabd43e 100644 --- a/llvm/test/CodeGen/ARM/call-graph-section-addrtaken.ll +++ b/llvm/test/CodeGen/ARM/call-graph-section-addrtaken.ll @@ -27,7 +27,7 @@ entry: !1 = !{i64 0, !"_ZTSFivE.generalized"} !2 = !{i64 0, !"_ZTSFviE.generalized"} -; CHECK: .section .llvm.callgraph,"o",%progbits,.text +; CHECK: .section .llvm.callgraph,"o",%llvm_call_graph,.text ;; Version ; CHECK-NEXT: .byte 0 ;; Flags -- Potential indirect target so LSB is set to 1. Other bits are 0. diff --git a/llvm/test/CodeGen/ARM/call-graph-section-assembly.ll b/llvm/test/CodeGen/ARM/call-graph-section-assembly.ll index ec8d5b8..3d3974e 100644 --- a/llvm/test/CodeGen/ARM/call-graph-section-assembly.ll +++ b/llvm/test/CodeGen/ARM/call-graph-section-assembly.ll @@ -36,7 +36,7 @@ entry: !4 = !{!5} !5 = !{i64 0, !"_ZTSFPvS_E.generalized"} -; CHECK: .section .llvm.callgraph,"o",%progbits,.text +; CHECK: .section .llvm.callgraph,"o",%llvm_call_graph,.text ;; Version ; CHECK-NEXT: .byte 0 ;; Flags diff --git a/llvm/test/CodeGen/DirectX/CBufferAccess/unused.ll b/llvm/test/CodeGen/DirectX/CBufferAccess/unused.ll new file mode 100644 index 0000000..8c0d82e --- /dev/null +++ b/llvm/test/CodeGen/DirectX/CBufferAccess/unused.ll @@ -0,0 +1,13 @@ +; RUN: opt -S -dxil-cbuffer-access -mtriple=dxil--shadermodel6.3-library %s | FileCheck %s +; Check that we correctly ignore cbuffers that were nulled out by optimizations. + +%__cblayout_CB = type <{ float }> +@CB.cb = local_unnamed_addr global target("dx.CBuffer", %__cblayout_CB) poison +@x = external local_unnamed_addr addrspace(2) global float, align 4 + +; CHECK-NOT: !hlsl.cbs = +!hlsl.cbs = !{!0, !1, !2} + +!0 = !{ptr @CB.cb, ptr addrspace(2) @x} +!1 = !{ptr @CB.cb, null} +!2 = !{null, null} diff --git a/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll b/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll index 245f764..7149cdb 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll @@ -32,9 +32,7 @@ define <16 x i16> @shuffle_v16i16(<16 x i16> %a) { ; CHECK: # %bb.0: ; CHECK-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_0) ; CHECK-NEXT: xvld $xr1, $a0, %pc_lo12(.LCPI2_0) -; CHECK-NEXT: xvpermi.d $xr2, $xr0, 78 -; CHECK-NEXT: xvshuf.w $xr1, $xr2, $xr0 -; CHECK-NEXT: xvori.b $xr0, $xr1, 0 +; CHECK-NEXT: xvperm.w $xr0, $xr0, $xr1 ; CHECK-NEXT: ret %shuffle = shufflevector <16 x i16> %a, <16 x i16> poison, <16 x i32> <i32 8, i32 9, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> ret <16 x i16> %shuffle @@ -55,9 +53,7 @@ define <16 x i16> @shuffle_v16i16_same_lane(<16 x i16> %a) { define <8 x i32> @shuffle_v8i32(<8 x i32> %a) { ; CHECK-LABEL: shuffle_v8i32: ; CHECK: # %bb.0: -; CHECK-NEXT: pcalau12i $a0, %pc_hi20(.LCPI4_0) -; CHECK-NEXT: xvld $xr1, $a0, %pc_lo12(.LCPI4_0) -; CHECK-NEXT: xvperm.w $xr0, $xr0, $xr1 +; CHECK-NEXT: xvpermi.d $xr0, $xr0, 226 ; CHECK-NEXT: ret %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 4, i32 5, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7> ret <8 x i32> %shuffle @@ -93,9 +89,7 @@ define <4 x i64> @shuffle_v4i64_same_lane(<4 x i64> %a) { define <8 x float> @shuffle_v8f32(<8 x float> %a) { ; CHECK-LABEL: shuffle_v8f32: ; CHECK: # %bb.0: -; CHECK-NEXT: pcalau12i $a0, %pc_hi20(.LCPI8_0) -; CHECK-NEXT: xvld $xr1, $a0, %pc_lo12(.LCPI8_0) -; CHECK-NEXT: xvperm.w $xr0, $xr0, $xr1 +; CHECK-NEXT: xvpermi.d $xr0, $xr0, 226 ; CHECK-NEXT: ret %shuffle = shufflevector <8 x float> %a, <8 x float> poison, <8 x i32> <i32 4, i32 5, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7> ret <8 x float> %shuffle diff --git a/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll index 061b2b0..abd00b6 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vfadd-sdnode.ll @@ -11,33 +11,80 @@ ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \ ; RUN: -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \ ; RUN: --check-prefixes=CHECK,ZVFHMIN +; RUN: llc -mtriple=riscv64 -mattr=+zvfh,+experimental-zvfbfa,+v \ +; RUN: -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \ +; RUN: --check-prefixes=CHECK,ZVFBFA define <vscale x 1 x bfloat> @vfadd_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb) { -; CHECK-LABEL: vfadd_vv_nxv1bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v9 -; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vfadd.vv v9, v9, v10 -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv1bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v9, v10 +; ZVFH-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv1bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v9, v10 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv1bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9 +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v9, v10 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 1 x bfloat> %va, %vb ret <vscale x 1 x bfloat> %vc } define <vscale x 1 x bfloat> @vfadd_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b) { -; CHECK-LABEL: vfadd_vf_nxv1bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vfadd.vf v9, v9, fa5 -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv1bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFH-NEXT: vfadd.vf v9, v9, fa5 +; ZVFH-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv1bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFHMIN-NEXT: vfadd.vf v9, v9, fa5 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv1bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vf v9, v9, fa5 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0 %splat = shufflevector <vscale x 1 x bfloat> %head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer %vc = fadd <vscale x 1 x bfloat> %va, %splat @@ -45,31 +92,75 @@ define <vscale x 1 x bfloat> @vfadd_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloa } define <vscale x 2 x bfloat> @vfadd_vv_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb) { -; CHECK-LABEL: vfadd_vv_nxv2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v9 -; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vfadd.vv v9, v9, v10 -; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv2bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v9, v10 +; ZVFH-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv2bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v9, v10 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv2bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9 +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v9, v10 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 2 x bfloat> %va, %vb ret <vscale x 2 x bfloat> %vc } define <vscale x 2 x bfloat> @vfadd_vf_nxv2bf16(<vscale x 2 x bfloat> %va, bfloat %b) { -; CHECK-LABEL: vfadd_vf_nxv2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vfadd.vf v9, v9, fa5 -; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv2bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFH-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFH-NEXT: vfadd.vf v9, v9, fa5 +; ZVFH-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv2bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFHMIN-NEXT: vfadd.vf v9, v9, fa5 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv2bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vf v9, v9, fa5 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 2 x bfloat> poison, bfloat %b, i32 0 %splat = shufflevector <vscale x 2 x bfloat> %head, <vscale x 2 x bfloat> poison, <vscale x 2 x i32> zeroinitializer %vc = fadd <vscale x 2 x bfloat> %va, %splat @@ -77,31 +168,75 @@ define <vscale x 2 x bfloat> @vfadd_vf_nxv2bf16(<vscale x 2 x bfloat> %va, bfloa } define <vscale x 4 x bfloat> @vfadd_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %vb) { -; CHECK-LABEL: vfadd_vv_nxv4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v9 -; CHECK-NEXT: vfwcvtbf16.f.f.v v12, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vfadd.vv v10, v12, v10 -; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v10 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv4bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v12, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFH-NEXT: vfadd.vv v10, v12, v10 +; ZVFH-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v10 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv4bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v12, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v10, v12, v10 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv4bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9 +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v10, v12, v10 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v10 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 4 x bfloat> %va, %vb ret <vscale x 4 x bfloat> %vc } define <vscale x 4 x bfloat> @vfadd_vf_nxv4bf16(<vscale x 4 x bfloat> %va, bfloat %b) { -; CHECK-LABEL: vfadd_vf_nxv4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vfadd.vf v10, v10, fa5 -; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v10 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv4bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFH-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFH-NEXT: vfadd.vf v10, v10, fa5 +; ZVFH-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v10 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv4bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFHMIN-NEXT: vfadd.vf v10, v10, fa5 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv4bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vf v10, v10, fa5 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v10 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 4 x bfloat> poison, bfloat %b, i32 0 %splat = shufflevector <vscale x 4 x bfloat> %head, <vscale x 4 x bfloat> poison, <vscale x 4 x i32> zeroinitializer %vc = fadd <vscale x 4 x bfloat> %va, %splat @@ -109,31 +244,75 @@ define <vscale x 4 x bfloat> @vfadd_vf_nxv4bf16(<vscale x 4 x bfloat> %va, bfloa } define <vscale x 8 x bfloat> @vfadd_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb) { -; CHECK-LABEL: vfadd_vv_nxv8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v12, v10 -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vfadd.vv v12, v16, v12 -; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v12 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv8bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v12, v10 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFH-NEXT: vfadd.vv v12, v16, v12 +; ZVFH-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v12 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv8bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v12, v10 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v12, v16, v12 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv8bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v10 +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vv v12, v16, v12 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v12 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 8 x bfloat> %va, %vb ret <vscale x 8 x bfloat> %vc } define <vscale x 8 x bfloat> @vfadd_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) { -; CHECK-LABEL: vfadd_vf_nxv8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v12, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vfadd.vf v12, v12, fa5 -; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v12 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv8bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFH-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v12, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFH-NEXT: vfadd.vf v12, v12, fa5 +; ZVFH-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v12 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv8bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v12, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFHMIN-NEXT: vfadd.vf v12, v12, fa5 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv8bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vf v12, v12, fa5 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v12 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0 %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer %vc = fadd <vscale x 8 x bfloat> %va, %splat @@ -141,16 +320,38 @@ define <vscale x 8 x bfloat> @vfadd_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloa } define <vscale x 8 x bfloat> @vfadd_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b) { -; CHECK-LABEL: vfadd_fv_nxv8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v12, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vfadd.vf v12, v12, fa5 -; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v12 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_fv_nxv8bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFH-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v12, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFH-NEXT: vfadd.vf v12, v12, fa5 +; ZVFH-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v12 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_fv_nxv8bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v12, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFHMIN-NEXT: vfadd.vf v12, v12, fa5 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_fv_nxv8bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vf v12, v12, fa5 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v12 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0 %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer %vc = fadd <vscale x 8 x bfloat> %splat, %va @@ -158,31 +359,75 @@ define <vscale x 8 x bfloat> @vfadd_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloa } define <vscale x 16 x bfloat> @vfadd_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %vb) { -; CHECK-LABEL: vfadd_vv_nxv16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v12 -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v24, v16 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v16 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv16bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v12 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v24, v16 +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv16bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v12 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v24, v16 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv16bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12 +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 16 x bfloat> %va, %vb ret <vscale x 16 x bfloat> %vc } define <vscale x 16 x bfloat> @vfadd_vf_nxv16bf16(<vscale x 16 x bfloat> %va, bfloat %b) { -; CHECK-LABEL: vfadd_vf_nxv16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vf v16, v16, fa5 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v16 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv16bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFH-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vf v16, v16, fa5 +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv16bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vf v16, v16, fa5 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv16bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vf v16, v16, fa5 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 16 x bfloat> poison, bfloat %b, i32 0 %splat = shufflevector <vscale x 16 x bfloat> %head, <vscale x 16 x bfloat> poison, <vscale x 16 x i32> zeroinitializer %vc = fadd <vscale x 16 x bfloat> %va, %splat @@ -190,78 +435,216 @@ define <vscale x 16 x bfloat> @vfadd_vf_nxv16bf16(<vscale x 16 x bfloat> %va, bf } define <vscale x 32 x bfloat> @vfadd_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %vb) { -; CHECK-LABEL: vfadd_vv_nxv32bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: addi sp, sp, -16 -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: sub sp, sp, a0 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb -; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v16 -; CHECK-NEXT: addi a0, sp, 16 -; CHECK-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill -; CHECK-NEXT: vfwcvtbf16.f.f.v v0, v8 -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v20 -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v12 -; CHECK-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v0, v0, v8 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v0 -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v16, v24 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v12, v16 -; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: add sp, sp, a0 -; CHECK-NEXT: .cfi_def_cfa sp, 16 -; CHECK-NEXT: addi sp, sp, 16 -; CHECK-NEXT: .cfi_def_cfa_offset 0 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv32bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: addi sp, sp, -16 +; ZVFH-NEXT: .cfi_def_cfa_offset 16 +; ZVFH-NEXT: csrr a0, vlenb +; ZVFH-NEXT: slli a0, a0, 3 +; ZVFH-NEXT: sub sp, sp, a0 +; ZVFH-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFH-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v16 +; ZVFH-NEXT: addi a0, sp, 16 +; ZVFH-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill +; ZVFH-NEXT: vfwcvtbf16.f.f.v v0, v8 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v20 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v12 +; ZVFH-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v0, v0, v8 +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v0 +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v16, v24 +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v12, v16 +; ZVFH-NEXT: csrr a0, vlenb +; ZVFH-NEXT: slli a0, a0, 3 +; ZVFH-NEXT: add sp, sp, a0 +; ZVFH-NEXT: .cfi_def_cfa sp, 16 +; ZVFH-NEXT: addi sp, sp, 16 +; ZVFH-NEXT: .cfi_def_cfa_offset 0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv32bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: addi sp, sp, -16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 16 +; ZVFHMIN-NEXT: csrr a0, vlenb +; ZVFHMIN-NEXT: slli a0, a0, 3 +; ZVFHMIN-NEXT: sub sp, sp, a0 +; ZVFHMIN-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v16 +; ZVFHMIN-NEXT: addi a0, sp, 16 +; ZVFHMIN-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v0, v8 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v20 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v12 +; ZVFHMIN-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v0, v0, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v0 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v16, v24 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v12, v16 +; ZVFHMIN-NEXT: csrr a0, vlenb +; ZVFHMIN-NEXT: slli a0, a0, 3 +; ZVFHMIN-NEXT: add sp, sp, a0 +; ZVFHMIN-NEXT: .cfi_def_cfa sp, 16 +; ZVFHMIN-NEXT: addi sp, sp, 16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv32bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: addi sp, sp, -16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 16 +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: sub sp, sp, a0 +; ZVFBFA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v16 +; ZVFBFA-NEXT: addi a0, sp, 16 +; ZVFBFA-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: vfwcvt.f.f.v v0, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v20 +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12 +; ZVFBFA-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v0, v0, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v0 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v24 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v12, v16 +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: add sp, sp, a0 +; ZVFBFA-NEXT: .cfi_def_cfa sp, 16 +; ZVFBFA-NEXT: addi sp, sp, 16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 0 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 32 x bfloat> %va, %vb ret <vscale x 32 x bfloat> %vc } define <vscale x 32 x bfloat> @vfadd_vf_nxv32bf16(<vscale x 32 x bfloat> %va, bfloat %b) { -; CHECK-LABEL: vfadd_vf_nxv32bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: addi sp, sp, -16 -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: sub sp, sp, a0 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb -; CHECK-NEXT: fmv.x.h a0, fa0 -; CHECK-NEXT: vsetvli a1, zero, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v8 -; CHECK-NEXT: addi a1, sp, 16 -; CHECK-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v12 -; CHECK-NEXT: vsetvli a1, zero, e16, m8, ta, ma -; CHECK-NEXT: vmv.v.x v8, a0 -; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v0, v8 -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v12 -; CHECK-NEXT: addi a0, sp, 16 -; CHECK-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v0, v8, v0 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v0 -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v24, v16 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v12, v16 -; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: add sp, sp, a0 -; CHECK-NEXT: .cfi_def_cfa sp, 16 -; CHECK-NEXT: addi sp, sp, 16 -; CHECK-NEXT: .cfi_def_cfa_offset 0 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv32bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: addi sp, sp, -16 +; ZVFH-NEXT: .cfi_def_cfa_offset 16 +; ZVFH-NEXT: csrr a0, vlenb +; ZVFH-NEXT: slli a0, a0, 3 +; ZVFH-NEXT: sub sp, sp, a0 +; ZVFH-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFH-NEXT: fmv.x.h a0, fa0 +; ZVFH-NEXT: vsetvli a1, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFH-NEXT: addi a1, sp, 16 +; ZVFH-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v12 +; ZVFH-NEXT: vsetvli a1, zero, e16, m8, ta, ma +; ZVFH-NEXT: vmv.v.x v8, a0 +; ZVFH-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v0, v8 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v12 +; ZVFH-NEXT: addi a0, sp, 16 +; ZVFH-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v0, v8, v0 +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v0 +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v24, v16 +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v12, v16 +; ZVFH-NEXT: csrr a0, vlenb +; ZVFH-NEXT: slli a0, a0, 3 +; ZVFH-NEXT: add sp, sp, a0 +; ZVFH-NEXT: .cfi_def_cfa sp, 16 +; ZVFH-NEXT: addi sp, sp, 16 +; ZVFH-NEXT: .cfi_def_cfa_offset 0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv32bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: addi sp, sp, -16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 16 +; ZVFHMIN-NEXT: csrr a0, vlenb +; ZVFHMIN-NEXT: slli a0, a0, 3 +; ZVFHMIN-NEXT: sub sp, sp, a0 +; ZVFHMIN-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFHMIN-NEXT: fmv.x.h a0, fa0 +; ZVFHMIN-NEXT: vsetvli a1, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFHMIN-NEXT: addi a1, sp, 16 +; ZVFHMIN-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v12 +; ZVFHMIN-NEXT: vsetvli a1, zero, e16, m8, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v8, a0 +; ZVFHMIN-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v0, v8 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v12 +; ZVFHMIN-NEXT: addi a0, sp, 16 +; ZVFHMIN-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v0, v8, v0 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v0 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v24, v16 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v12, v16 +; ZVFHMIN-NEXT: csrr a0, vlenb +; ZVFHMIN-NEXT: slli a0, a0, 3 +; ZVFHMIN-NEXT: add sp, sp, a0 +; ZVFHMIN-NEXT: .cfi_def_cfa sp, 16 +; ZVFHMIN-NEXT: addi sp, sp, 16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv32bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: addi sp, sp, -16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 16 +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: sub sp, sp, a0 +; ZVFBFA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFBFA-NEXT: fmv.x.h a0, fa0 +; ZVFBFA-NEXT: vsetvli a1, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8 +; ZVFBFA-NEXT: addi a1, sp, 16 +; ZVFBFA-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v12 +; ZVFBFA-NEXT: vsetvli a1, zero, e16alt, m8, ta, ma +; ZVFBFA-NEXT: vmv.v.x v8, a0 +; ZVFBFA-NEXT: vsetvli a0, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v0, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12 +; ZVFBFA-NEXT: addi a0, sp, 16 +; ZVFBFA-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v0, v8, v0 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v0 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v12, v16 +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: add sp, sp, a0 +; ZVFBFA-NEXT: .cfi_def_cfa sp, 16 +; ZVFBFA-NEXT: addi sp, sp, 16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 0 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 32 x bfloat> poison, bfloat %b, i32 0 %splat = shufflevector <vscale x 32 x bfloat> %head, <vscale x 32 x bfloat> poison, <vscale x 32 x i32> zeroinitializer %vc = fadd <vscale x 32 x bfloat> %va, %splat @@ -285,6 +668,12 @@ define <vscale x 1 x half> @vfadd_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv1f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; ZVFBFA-NEXT: vfadd.vv v8, v8, v9 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 1 x half> %va, %vb ret <vscale x 1 x half> %vc } @@ -306,6 +695,12 @@ define <vscale x 1 x half> @vfadd_vf_nxv1f16(<vscale x 1 x half> %va, half %b) { ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv1f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; ZVFBFA-NEXT: vfadd.vf v8, v8, fa0 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 1 x half> poison, half %b, i32 0 %splat = shufflevector <vscale x 1 x half> %head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer %vc = fadd <vscale x 1 x half> %va, %splat @@ -329,6 +724,12 @@ define <vscale x 2 x half> @vfadd_vv_nxv2f16(<vscale x 2 x half> %va, <vscale x ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv2f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v8, v8, v9 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 2 x half> %va, %vb ret <vscale x 2 x half> %vc } @@ -350,6 +751,12 @@ define <vscale x 2 x half> @vfadd_vf_nxv2f16(<vscale x 2 x half> %va, half %b) { ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv2f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vf v8, v8, fa0 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 2 x half> poison, half %b, i32 0 %splat = shufflevector <vscale x 2 x half> %head, <vscale x 2 x half> poison, <vscale x 2 x i32> zeroinitializer %vc = fadd <vscale x 2 x half> %va, %splat @@ -373,6 +780,12 @@ define <vscale x 4 x half> @vfadd_vv_nxv4f16(<vscale x 4 x half> %va, <vscale x ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v10 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv4f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vv v8, v8, v9 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 4 x half> %va, %vb ret <vscale x 4 x half> %vc } @@ -394,6 +807,12 @@ define <vscale x 4 x half> @vfadd_vf_nxv4f16(<vscale x 4 x half> %va, half %b) { ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v10 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv4f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vf v8, v8, fa0 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 4 x half> poison, half %b, i32 0 %splat = shufflevector <vscale x 4 x half> %head, <vscale x 4 x half> poison, <vscale x 4 x i32> zeroinitializer %vc = fadd <vscale x 4 x half> %va, %splat @@ -417,6 +836,12 @@ define <vscale x 8 x half> @vfadd_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v12 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv8f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v8, v8, v10 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 8 x half> %va, %vb ret <vscale x 8 x half> %vc } @@ -438,6 +863,12 @@ define <vscale x 8 x half> @vfadd_vf_nxv8f16(<vscale x 8 x half> %va, half %b) { ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v12 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv8f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vf v8, v8, fa0 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 8 x half> poison, half %b, i32 0 %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer %vc = fadd <vscale x 8 x half> %va, %splat @@ -461,6 +892,12 @@ define <vscale x 8 x half> @vfadd_fv_nxv8f16(<vscale x 8 x half> %va, half %b) { ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v12 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_fv_nxv8f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vf v8, v8, fa0 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 8 x half> poison, half %b, i32 0 %splat = shufflevector <vscale x 8 x half> %head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer %vc = fadd <vscale x 8 x half> %splat, %va @@ -484,6 +921,12 @@ define <vscale x 16 x half> @vfadd_vv_nxv16f16(<vscale x 16 x half> %va, <vscale ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v16 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv16f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vv v8, v8, v12 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 16 x half> %va, %vb ret <vscale x 16 x half> %vc } @@ -505,6 +948,12 @@ define <vscale x 16 x half> @vfadd_vf_nxv16f16(<vscale x 16 x half> %va, half %b ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v16 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv16f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vf v8, v8, fa0 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 16 x half> poison, half %b, i32 0 %splat = shufflevector <vscale x 16 x half> %head, <vscale x 16 x half> poison, <vscale x 16 x i32> zeroinitializer %vc = fadd <vscale x 16 x half> %va, %splat @@ -549,6 +998,12 @@ define <vscale x 32 x half> @vfadd_vv_nxv32f16(<vscale x 32 x half> %va, <vscale ; ZVFHMIN-NEXT: addi sp, sp, 16 ; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv32f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v8, v8, v16 +; ZVFBFA-NEXT: ret %vc = fadd <vscale x 32 x half> %va, %vb ret <vscale x 32 x half> %vc } @@ -596,6 +1051,12 @@ define <vscale x 32 x half> @vfadd_vf_nxv32f16(<vscale x 32 x half> %va, half %b ; ZVFHMIN-NEXT: addi sp, sp, 16 ; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv32f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli a0, zero, e16, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vf v8, v8, fa0 +; ZVFBFA-NEXT: ret %head = insertelement <vscale x 32 x half> poison, half %b, i32 0 %splat = shufflevector <vscale x 32 x half> %head, <vscale x 32 x half> poison, <vscale x 32 x i32> zeroinitializer %vc = fadd <vscale x 32 x half> %va, %splat diff --git a/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll index 32e3d6b..633a201 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll @@ -11,52 +11,125 @@ ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \ ; RUN: -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \ ; RUN: --check-prefixes=CHECK,ZVFHMIN +; RUN: llc -mtriple=riscv64 -mattr=+zvfhmin,+experimental-zvfbfa,+v \ +; RUN: -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \ +; RUN: --check-prefixes=CHECK,ZVFBFA declare <vscale x 1 x bfloat> @llvm.vp.fadd.nxv1bf16(<vscale x 1 x bfloat>, <vscale x 1 x bfloat>, <vscale x 1 x i1>, i32) define <vscale x 1 x bfloat> @vfadd_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %b, <vscale x 1 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv1bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v9, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vfadd.vv v9, v9, v10, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv1bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v9, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v9, v8, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v9, v10, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv1bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v9, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v9, v8, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v9, v10, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv1bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v9, v10, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9, v0.t +; ZVFBFA-NEXT: ret %v = call <vscale x 1 x bfloat> @llvm.vp.fadd.nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %b, <vscale x 1 x i1> %m, i32 %evl) ret <vscale x 1 x bfloat> %v } define <vscale x 1 x bfloat> @vfadd_vv_nxv1bf16_unmasked(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv1bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v9 -; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vfadd.vv v9, v9, v10 -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv1bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v9, v10 +; ZVFH-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv1bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v9, v10 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv1bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9 +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v9, v10 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %v = call <vscale x 1 x bfloat> @llvm.vp.fadd.nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %b, <vscale x 1 x i1> splat (i1 true), i32 %evl) ret <vscale x 1 x bfloat> %v } define <vscale x 1 x bfloat> @vfadd_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv1bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v9, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vfadd.vv v9, v10, v8, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv1bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFH-NEXT: vmv.v.x v9, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v9, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v10, v8, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv1bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v9, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v9, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v10, v8, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv1bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v10, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer %v = call <vscale x 1 x bfloat> @llvm.vp.fadd.nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb, <vscale x 1 x i1> %m, i32 %evl) @@ -64,18 +137,44 @@ define <vscale x 1 x bfloat> @vfadd_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloa } define <vscale x 1 x bfloat> @vfadd_vf_nxv1bf16_commute(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv1bf16_commute: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v9, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vfadd.vv v9, v8, v10, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv1bf16_commute: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFH-NEXT: vmv.v.x v9, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v9, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v8, v10, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv1bf16_commute: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v9, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v9, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v8, v10, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv1bf16_commute: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v8, v10, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer %v = call <vscale x 1 x bfloat> @llvm.vp.fadd.nxv1bf16(<vscale x 1 x bfloat> %vb, <vscale x 1 x bfloat> %va, <vscale x 1 x i1> %m, i32 %evl) @@ -83,18 +182,44 @@ define <vscale x 1 x bfloat> @vfadd_vf_nxv1bf16_commute(<vscale x 1 x bfloat> %v } define <vscale x 1 x bfloat> @vfadd_vf_nxv1bf16_unmasked(<vscale x 1 x bfloat> %va, bfloat %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv1bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8 -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v9 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vfadd.vv v9, v10, v8 -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv1bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFH-NEXT: vmv.v.x v9, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v8 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v9 +; ZVFH-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v10, v8 +; ZVFH-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv1bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v9, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v8 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v9 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v10, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv1bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v10, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer %v = call <vscale x 1 x bfloat> @llvm.vp.fadd.nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb, <vscale x 1 x i1> splat (i1 true), i32 %evl) @@ -102,18 +227,44 @@ define <vscale x 1 x bfloat> @vfadd_vf_nxv1bf16_unmasked(<vscale x 1 x bfloat> % } define <vscale x 1 x bfloat> @vfadd_vf_nxv1bf16_unmasked_commute(<vscale x 1 x bfloat> %va, bfloat %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv1bf16_unmasked_commute: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8 -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v9 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vfadd.vv v9, v8, v10 -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv1bf16_unmasked_commute: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFH-NEXT: vmv.v.x v9, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v8 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v9 +; ZVFH-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v8, v10 +; ZVFH-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv1bf16_unmasked_commute: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v9, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v8 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v9 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v8, v10 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv1bf16_unmasked_commute: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v8, v10 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer %v = call <vscale x 1 x bfloat> @llvm.vp.fadd.nxv1bf16(<vscale x 1 x bfloat> %vb, <vscale x 1 x bfloat> %va, <vscale x 1 x i1> splat (i1 true), i32 %evl) @@ -123,48 +274,118 @@ define <vscale x 1 x bfloat> @vfadd_vf_nxv1bf16_unmasked_commute(<vscale x 1 x b declare <vscale x 2 x bfloat> @llvm.vp.fadd.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>, <vscale x 2 x i1>, i32) define <vscale x 2 x bfloat> @vfadd_vv_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %b, <vscale x 2 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v9, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vfadd.vv v9, v9, v10, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv2bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v9, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v9, v8, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v9, v10, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv2bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v9, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v9, v8, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v9, v10, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv2bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v9, v10, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9, v0.t +; ZVFBFA-NEXT: ret %v = call <vscale x 2 x bfloat> @llvm.vp.fadd.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %b, <vscale x 2 x i1> %m, i32 %evl) ret <vscale x 2 x bfloat> %v } define <vscale x 2 x bfloat> @vfadd_vv_nxv2bf16_unmasked(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv2bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v9 -; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vfadd.vv v9, v9, v10 -; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv2bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v9, v10 +; ZVFH-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv2bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v9, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v9, v10 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv2bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9 +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v9, v10 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %v = call <vscale x 2 x bfloat> @llvm.vp.fadd.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %b, <vscale x 2 x i1> splat (i1 true), i32 %evl) ret <vscale x 2 x bfloat> %v } define <vscale x 2 x bfloat> @vfadd_vf_nxv2bf16(<vscale x 2 x bfloat> %va, bfloat %b, <vscale x 2 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v9, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v9, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vfadd.vv v9, v10, v8, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv2bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFH-NEXT: vmv.v.x v9, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v9, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v10, v8, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv2bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v9, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v9, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v10, v8, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv2bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v10, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 2 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 2 x bfloat> %elt.head, <vscale x 2 x bfloat> poison, <vscale x 2 x i32> zeroinitializer %v = call <vscale x 2 x bfloat> @llvm.vp.fadd.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %m, i32 %evl) @@ -172,18 +393,44 @@ define <vscale x 2 x bfloat> @vfadd_vf_nxv2bf16(<vscale x 2 x bfloat> %va, bfloa } define <vscale x 2 x bfloat> @vfadd_vf_nxv2bf16_unmasked(<vscale x 2 x bfloat> %va, bfloat %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv2bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v9, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8 -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v9 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vfadd.vv v9, v10, v8 -; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v9 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv2bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFH-NEXT: vmv.v.x v9, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v8 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v9 +; ZVFH-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFH-NEXT: vfadd.vv v9, v10, v8 +; ZVFH-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv2bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v9, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v8 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v9 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v9, v10, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv2bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v10, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, mf2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 2 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 2 x bfloat> %elt.head, <vscale x 2 x bfloat> poison, <vscale x 2 x i32> zeroinitializer %v = call <vscale x 2 x bfloat> @llvm.vp.fadd.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> splat (i1 true), i32 %evl) @@ -193,48 +440,118 @@ define <vscale x 2 x bfloat> @vfadd_vf_nxv2bf16_unmasked(<vscale x 2 x bfloat> % declare <vscale x 4 x bfloat> @llvm.vp.fadd.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>, <vscale x 4 x i1>, i32) define <vscale x 4 x bfloat> @vfadd_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %b, <vscale x 4 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v9, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v12, v8, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vfadd.vv v10, v12, v10, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv4bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v9, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v12, v8, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFH-NEXT: vfadd.vv v10, v12, v10, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv4bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v9, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v12, v8, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v10, v12, v10, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv4bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v10, v12, v10, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v10, v0.t +; ZVFBFA-NEXT: ret %v = call <vscale x 4 x bfloat> @llvm.vp.fadd.nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %b, <vscale x 4 x i1> %m, i32 %evl) ret <vscale x 4 x bfloat> %v } define <vscale x 4 x bfloat> @vfadd_vv_nxv4bf16_unmasked(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv4bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v9 -; CHECK-NEXT: vfwcvtbf16.f.f.v v12, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vfadd.vv v10, v12, v10 -; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v10 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv4bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v12, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFH-NEXT: vfadd.vv v10, v12, v10 +; ZVFH-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v10 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv4bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v9 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v12, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v10, v12, v10 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv4bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9 +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v10, v12, v10 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v10 +; ZVFBFA-NEXT: ret %v = call <vscale x 4 x bfloat> @llvm.vp.fadd.nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %b, <vscale x 4 x i1> splat (i1 true), i32 %evl) ret <vscale x 4 x bfloat> %v } define <vscale x 4 x bfloat> @vfadd_vf_nxv4bf16(<vscale x 4 x bfloat> %va, bfloat %b, <vscale x 4 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma -; CHECK-NEXT: vmv.v.x v12, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v12, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vfadd.vv v10, v10, v8, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv4bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFH-NEXT: vmv.v.x v12, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v12, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFH-NEXT: vfadd.vv v10, v10, v8, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv4bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v12, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v8, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v12, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v10, v10, v8, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv4bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vmv.v.x v12, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v10, v10, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v10, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 4 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 4 x bfloat> %elt.head, <vscale x 4 x bfloat> poison, <vscale x 4 x i32> zeroinitializer %v = call <vscale x 4 x bfloat> @llvm.vp.fadd.nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %vb, <vscale x 4 x i1> %m, i32 %evl) @@ -242,18 +559,44 @@ define <vscale x 4 x bfloat> @vfadd_vf_nxv4bf16(<vscale x 4 x bfloat> %va, bfloa } define <vscale x 4 x bfloat> @vfadd_vf_nxv4bf16_unmasked(<vscale x 4 x bfloat> %va, bfloat %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv4bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma -; CHECK-NEXT: vmv.v.x v12, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8 -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v12 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vfadd.vv v10, v10, v8 -; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v10 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv4bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFH-NEXT: vmv.v.x v12, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v10, v8 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v12 +; ZVFH-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFH-NEXT: vfadd.vv v10, v10, v8 +; ZVFH-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v10 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv4bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v12, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v10, v8 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v12 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v10, v10, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv4bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vmv.v.x v12, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v12 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v10, v10, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m1, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v10 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 4 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 4 x bfloat> %elt.head, <vscale x 4 x bfloat> poison, <vscale x 4 x i32> zeroinitializer %v = call <vscale x 4 x bfloat> @llvm.vp.fadd.nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %vb, <vscale x 4 x i1> splat (i1 true), i32 %evl) @@ -263,48 +606,118 @@ define <vscale x 4 x bfloat> @vfadd_vf_nxv4bf16_unmasked(<vscale x 4 x bfloat> % declare <vscale x 8 x bfloat> @llvm.vp.fadd.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x i1>, i32) define <vscale x 8 x bfloat> @vfadd_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m2, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v12, v10, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v8, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vfadd.vv v12, v16, v12, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv8bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v12, v10, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v8, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFH-NEXT: vfadd.vv v12, v16, v12, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv8bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v12, v10, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v8, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v12, v16, v12, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv8bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v10, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vv v12, v16, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v12, v0.t +; ZVFBFA-NEXT: ret %v = call <vscale x 8 x bfloat> @llvm.vp.fadd.nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %m, i32 %evl) ret <vscale x 8 x bfloat> %v } define <vscale x 8 x bfloat> @vfadd_vv_nxv8bf16_unmasked(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv8bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m2, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v12, v10 -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vfadd.vv v12, v16, v12 -; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v12 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv8bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v12, v10 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFH-NEXT: vfadd.vv v12, v16, v12 +; ZVFH-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v12 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv8bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v12, v10 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v12, v16, v12 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv8bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v10 +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vv v12, v16, v12 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v12 +; ZVFBFA-NEXT: ret %v = call <vscale x 8 x bfloat> @llvm.vp.fadd.nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> splat (i1 true), i32 %evl) ret <vscale x 8 x bfloat> %v } define <vscale x 8 x bfloat> @vfadd_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, m2, ta, ma -; CHECK-NEXT: vmv.v.x v16, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v12, v8, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v16, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vfadd.vv v12, v12, v8, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv8bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFH-NEXT: vmv.v.x v16, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v12, v8, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v16, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFH-NEXT: vfadd.vv v12, v12, v8, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv8bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v16, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v12, v8, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v16, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v12, v12, v8, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv8bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vmv.v.x v16, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v16, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vv v12, v12, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v12, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 8 x bfloat> %elt.head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer %v = call <vscale x 8 x bfloat> @llvm.vp.fadd.nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb, <vscale x 8 x i1> %m, i32 %evl) @@ -312,18 +725,44 @@ define <vscale x 8 x bfloat> @vfadd_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloa } define <vscale x 8 x bfloat> @vfadd_vf_nxv8bf16_unmasked(<vscale x 8 x bfloat> %va, bfloat %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv8bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, m2, ta, ma -; CHECK-NEXT: vmv.v.x v16, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v12, v8 -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v16 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vfadd.vv v12, v12, v8 -; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v12 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv8bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFH-NEXT: vmv.v.x v16, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v12, v8 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v16 +; ZVFH-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFH-NEXT: vfadd.vv v12, v12, v8 +; ZVFH-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v12 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv8bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v16, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v12, v8 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v16 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v12, v12, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv8bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vmv.v.x v16, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v16 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vv v12, v12, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v12 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 8 x bfloat> %elt.head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer %v = call <vscale x 8 x bfloat> @llvm.vp.fadd.nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb, <vscale x 8 x i1> splat (i1 true), i32 %evl) @@ -333,48 +772,118 @@ define <vscale x 8 x bfloat> @vfadd_vf_nxv8bf16_unmasked(<vscale x 8 x bfloat> % declare <vscale x 16 x bfloat> @llvm.vp.fadd.nxv16bf16(<vscale x 16 x bfloat>, <vscale x 16 x bfloat>, <vscale x 16 x i1>, i32) define <vscale x 16 x bfloat> @vfadd_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %b, <vscale x 16 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v8, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v24, v16, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv16bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v8, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv16bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v8, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv16bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16, v0.t +; ZVFBFA-NEXT: ret %v = call <vscale x 16 x bfloat> @llvm.vp.fadd.nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %b, <vscale x 16 x i1> %m, i32 %evl) ret <vscale x 16 x bfloat> %v } define <vscale x 16 x bfloat> @vfadd_vv_nxv16bf16_unmasked(<vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv16bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v12 -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v24, v16 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v16 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv16bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v12 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v24, v16 +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv16bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v12 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v24, v16 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv16bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12 +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16 +; ZVFBFA-NEXT: ret %v = call <vscale x 16 x bfloat> @llvm.vp.fadd.nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %b, <vscale x 16 x i1> splat (i1 true), i32 %evl) ret <vscale x 16 x bfloat> %v } define <vscale x 16 x bfloat> @vfadd_vf_nxv16bf16(<vscale x 16 x bfloat> %va, bfloat %b, <vscale x 16 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, m4, ta, ma -; CHECK-NEXT: vmv.v.x v24, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v8, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v24, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v16, v8, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv16bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFH-NEXT: vmv.v.x v24, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v8, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v24, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v16, v8, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv16bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v24, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v8, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v24, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v16, v8, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv16bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v24, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v24, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 16 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 16 x bfloat> %elt.head, <vscale x 16 x bfloat> poison, <vscale x 16 x i32> zeroinitializer %v = call <vscale x 16 x bfloat> @llvm.vp.fadd.nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %vb, <vscale x 16 x i1> %m, i32 %evl) @@ -382,18 +891,44 @@ define <vscale x 16 x bfloat> @vfadd_vf_nxv16bf16(<vscale x 16 x bfloat> %va, bf } define <vscale x 16 x bfloat> @vfadd_vf_nxv16bf16_unmasked(<vscale x 16 x bfloat> %va, bfloat %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv16bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: vsetvli zero, a0, e16, m4, ta, ma -; CHECK-NEXT: vmv.v.x v24, a1 -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v8 -; CHECK-NEXT: vfwcvtbf16.f.f.v v8, v24 -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v16, v8 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v16 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv16bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFH-NEXT: vmv.v.x v24, a1 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v8, v24 +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v16, v8 +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv16bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFHMIN-NEXT: vmv.v.x v24, a1 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v8, v24 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v16, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv16bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v24, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v24 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 16 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 16 x bfloat> %elt.head, <vscale x 16 x bfloat> poison, <vscale x 16 x i32> zeroinitializer %v = call <vscale x 16 x bfloat> @llvm.vp.fadd.nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %vb, <vscale x 16 x i1> splat (i1 true), i32 %evl) @@ -403,173 +938,493 @@ define <vscale x 16 x bfloat> @vfadd_vf_nxv16bf16_unmasked(<vscale x 16 x bfloat declare <vscale x 32 x bfloat> @llvm.vp.fadd.nxv32bf16(<vscale x 32 x bfloat>, <vscale x 32 x bfloat>, <vscale x 32 x i1>, i32) define <vscale x 32 x bfloat> @vfadd_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %b, <vscale x 32 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv32bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: addi sp, sp, -16 -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: slli a1, a1, 3 -; CHECK-NEXT: sub sp, sp, a1 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb -; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma -; CHECK-NEXT: vmv1r.v v7, v0 -; CHECK-NEXT: csrr a2, vlenb -; CHECK-NEXT: slli a1, a2, 1 -; CHECK-NEXT: srli a2, a2, 2 -; CHECK-NEXT: sub a3, a0, a1 -; CHECK-NEXT: vslidedown.vx v0, v0, a2 -; CHECK-NEXT: sltu a2, a0, a3 -; CHECK-NEXT: addi a2, a2, -1 -; CHECK-NEXT: and a2, a2, a3 -; CHECK-NEXT: addi a3, sp, 16 -; CHECK-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill -; CHECK-NEXT: vsetvli zero, a2, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v20, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v16, v24, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t -; CHECK-NEXT: bltu a0, a1, .LBB22_2 -; CHECK-NEXT: # %bb.1: -; CHECK-NEXT: mv a0, a1 -; CHECK-NEXT: .LBB22_2: -; CHECK-NEXT: vmv1r.v v0, v7 -; CHECK-NEXT: addi a1, sp, 16 -; CHECK-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload -; CHECK-NEXT: vsetvli zero, a0, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v24, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v8, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v24, v16, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t -; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: add sp, sp, a0 -; CHECK-NEXT: .cfi_def_cfa sp, 16 -; CHECK-NEXT: addi sp, sp, 16 -; CHECK-NEXT: .cfi_def_cfa_offset 0 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv32bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: addi sp, sp, -16 +; ZVFH-NEXT: .cfi_def_cfa_offset 16 +; ZVFH-NEXT: csrr a1, vlenb +; ZVFH-NEXT: slli a1, a1, 3 +; ZVFH-NEXT: sub sp, sp, a1 +; ZVFH-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFH-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; ZVFH-NEXT: vmv1r.v v7, v0 +; ZVFH-NEXT: csrr a2, vlenb +; ZVFH-NEXT: slli a1, a2, 1 +; ZVFH-NEXT: srli a2, a2, 2 +; ZVFH-NEXT: sub a3, a0, a1 +; ZVFH-NEXT: vslidedown.vx v0, v0, a2 +; ZVFH-NEXT: sltu a2, a0, a3 +; ZVFH-NEXT: addi a2, a2, -1 +; ZVFH-NEXT: and a2, a2, a3 +; ZVFH-NEXT: addi a3, sp, 16 +; ZVFH-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFH-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v20, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t +; ZVFH-NEXT: bltu a0, a1, .LBB22_2 +; ZVFH-NEXT: # %bb.1: +; ZVFH-NEXT: mv a0, a1 +; ZVFH-NEXT: .LBB22_2: +; ZVFH-NEXT: vmv1r.v v0, v7 +; ZVFH-NEXT: addi a1, sp, 16 +; ZVFH-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; ZVFH-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v24, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v8, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; ZVFH-NEXT: csrr a0, vlenb +; ZVFH-NEXT: slli a0, a0, 3 +; ZVFH-NEXT: add sp, sp, a0 +; ZVFH-NEXT: .cfi_def_cfa sp, 16 +; ZVFH-NEXT: addi sp, sp, 16 +; ZVFH-NEXT: .cfi_def_cfa_offset 0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv32bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: addi sp, sp, -16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 16 +; ZVFHMIN-NEXT: csrr a1, vlenb +; ZVFHMIN-NEXT: slli a1, a1, 3 +; ZVFHMIN-NEXT: sub sp, sp, a1 +; ZVFHMIN-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFHMIN-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; ZVFHMIN-NEXT: vmv1r.v v7, v0 +; ZVFHMIN-NEXT: csrr a2, vlenb +; ZVFHMIN-NEXT: slli a1, a2, 1 +; ZVFHMIN-NEXT: srli a2, a2, 2 +; ZVFHMIN-NEXT: sub a3, a0, a1 +; ZVFHMIN-NEXT: vslidedown.vx v0, v0, a2 +; ZVFHMIN-NEXT: sltu a2, a0, a3 +; ZVFHMIN-NEXT: addi a2, a2, -1 +; ZVFHMIN-NEXT: and a2, a2, a3 +; ZVFHMIN-NEXT: addi a3, sp, 16 +; ZVFHMIN-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFHMIN-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v20, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t +; ZVFHMIN-NEXT: bltu a0, a1, .LBB22_2 +; ZVFHMIN-NEXT: # %bb.1: +; ZVFHMIN-NEXT: mv a0, a1 +; ZVFHMIN-NEXT: .LBB22_2: +; ZVFHMIN-NEXT: vmv1r.v v0, v7 +; ZVFHMIN-NEXT: addi a1, sp, 16 +; ZVFHMIN-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v24, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v8, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; ZVFHMIN-NEXT: csrr a0, vlenb +; ZVFHMIN-NEXT: slli a0, a0, 3 +; ZVFHMIN-NEXT: add sp, sp, a0 +; ZVFHMIN-NEXT: .cfi_def_cfa sp, 16 +; ZVFHMIN-NEXT: addi sp, sp, 16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv32bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: addi sp, sp, -16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 16 +; ZVFBFA-NEXT: csrr a1, vlenb +; ZVFBFA-NEXT: slli a1, a1, 3 +; ZVFBFA-NEXT: sub sp, sp, a1 +; ZVFBFA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFBFA-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; ZVFBFA-NEXT: vmv1r.v v7, v0 +; ZVFBFA-NEXT: csrr a2, vlenb +; ZVFBFA-NEXT: slli a1, a2, 1 +; ZVFBFA-NEXT: srli a2, a2, 2 +; ZVFBFA-NEXT: sub a3, a0, a1 +; ZVFBFA-NEXT: vslidedown.vx v0, v0, a2 +; ZVFBFA-NEXT: sltu a2, a0, a3 +; ZVFBFA-NEXT: addi a2, a2, -1 +; ZVFBFA-NEXT: and a2, a2, a3 +; ZVFBFA-NEXT: addi a3, sp, 16 +; ZVFBFA-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: vsetvli zero, a2, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v20, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v12, v16, v0.t +; ZVFBFA-NEXT: bltu a0, a1, .LBB22_2 +; ZVFBFA-NEXT: # %bb.1: +; ZVFBFA-NEXT: mv a0, a1 +; ZVFBFA-NEXT: .LBB22_2: +; ZVFBFA-NEXT: vmv1r.v v0, v7 +; ZVFBFA-NEXT: addi a1, sp, 16 +; ZVFBFA-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v24, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16, v0.t +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: add sp, sp, a0 +; ZVFBFA-NEXT: .cfi_def_cfa sp, 16 +; ZVFBFA-NEXT: addi sp, sp, 16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 0 +; ZVFBFA-NEXT: ret %v = call <vscale x 32 x bfloat> @llvm.vp.fadd.nxv32bf16(<vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %b, <vscale x 32 x i1> %m, i32 %evl) ret <vscale x 32 x bfloat> %v } define <vscale x 32 x bfloat> @vfadd_vv_nxv32bf16_unmasked(<vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vv_nxv32bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: addi sp, sp, -16 -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: slli a1, a1, 3 -; CHECK-NEXT: sub sp, sp, a1 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb -; CHECK-NEXT: csrr a2, vlenb -; CHECK-NEXT: vsetvli a1, zero, e8, m4, ta, ma -; CHECK-NEXT: vmset.m v24 -; CHECK-NEXT: slli a1, a2, 1 -; CHECK-NEXT: srli a2, a2, 2 -; CHECK-NEXT: sub a3, a0, a1 -; CHECK-NEXT: vsetvli a4, zero, e8, mf2, ta, ma -; CHECK-NEXT: vslidedown.vx v0, v24, a2 -; CHECK-NEXT: sltu a2, a0, a3 -; CHECK-NEXT: addi a2, a2, -1 -; CHECK-NEXT: and a2, a2, a3 -; CHECK-NEXT: addi a3, sp, 16 -; CHECK-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill -; CHECK-NEXT: vsetvli zero, a2, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v20, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v16, v24, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t -; CHECK-NEXT: bltu a0, a1, .LBB23_2 -; CHECK-NEXT: # %bb.1: -; CHECK-NEXT: mv a0, a1 -; CHECK-NEXT: .LBB23_2: -; CHECK-NEXT: addi a1, sp, 16 -; CHECK-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload -; CHECK-NEXT: vsetvli zero, a0, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v24 -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v8 -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v24, v16 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v16 -; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: add sp, sp, a0 -; CHECK-NEXT: .cfi_def_cfa sp, 16 -; CHECK-NEXT: addi sp, sp, 16 -; CHECK-NEXT: .cfi_def_cfa_offset 0 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vv_nxv32bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: addi sp, sp, -16 +; ZVFH-NEXT: .cfi_def_cfa_offset 16 +; ZVFH-NEXT: csrr a1, vlenb +; ZVFH-NEXT: slli a1, a1, 3 +; ZVFH-NEXT: sub sp, sp, a1 +; ZVFH-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFH-NEXT: csrr a2, vlenb +; ZVFH-NEXT: vsetvli a1, zero, e8, m4, ta, ma +; ZVFH-NEXT: vmset.m v24 +; ZVFH-NEXT: slli a1, a2, 1 +; ZVFH-NEXT: srli a2, a2, 2 +; ZVFH-NEXT: sub a3, a0, a1 +; ZVFH-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFH-NEXT: vslidedown.vx v0, v24, a2 +; ZVFH-NEXT: sltu a2, a0, a3 +; ZVFH-NEXT: addi a2, a2, -1 +; ZVFH-NEXT: and a2, a2, a3 +; ZVFH-NEXT: addi a3, sp, 16 +; ZVFH-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFH-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v20, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t +; ZVFH-NEXT: bltu a0, a1, .LBB23_2 +; ZVFH-NEXT: # %bb.1: +; ZVFH-NEXT: mv a0, a1 +; ZVFH-NEXT: .LBB23_2: +; ZVFH-NEXT: addi a1, sp, 16 +; ZVFH-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; ZVFH-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v24 +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v8 +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v24, v16 +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFH-NEXT: csrr a0, vlenb +; ZVFH-NEXT: slli a0, a0, 3 +; ZVFH-NEXT: add sp, sp, a0 +; ZVFH-NEXT: .cfi_def_cfa sp, 16 +; ZVFH-NEXT: addi sp, sp, 16 +; ZVFH-NEXT: .cfi_def_cfa_offset 0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vv_nxv32bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: addi sp, sp, -16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 16 +; ZVFHMIN-NEXT: csrr a1, vlenb +; ZVFHMIN-NEXT: slli a1, a1, 3 +; ZVFHMIN-NEXT: sub sp, sp, a1 +; ZVFHMIN-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFHMIN-NEXT: csrr a2, vlenb +; ZVFHMIN-NEXT: vsetvli a1, zero, e8, m4, ta, ma +; ZVFHMIN-NEXT: vmset.m v24 +; ZVFHMIN-NEXT: slli a1, a2, 1 +; ZVFHMIN-NEXT: srli a2, a2, 2 +; ZVFHMIN-NEXT: sub a3, a0, a1 +; ZVFHMIN-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFHMIN-NEXT: vslidedown.vx v0, v24, a2 +; ZVFHMIN-NEXT: sltu a2, a0, a3 +; ZVFHMIN-NEXT: addi a2, a2, -1 +; ZVFHMIN-NEXT: and a2, a2, a3 +; ZVFHMIN-NEXT: addi a3, sp, 16 +; ZVFHMIN-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFHMIN-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v20, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t +; ZVFHMIN-NEXT: bltu a0, a1, .LBB23_2 +; ZVFHMIN-NEXT: # %bb.1: +; ZVFHMIN-NEXT: mv a0, a1 +; ZVFHMIN-NEXT: .LBB23_2: +; ZVFHMIN-NEXT: addi a1, sp, 16 +; ZVFHMIN-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v24 +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v8 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v24, v16 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFHMIN-NEXT: csrr a0, vlenb +; ZVFHMIN-NEXT: slli a0, a0, 3 +; ZVFHMIN-NEXT: add sp, sp, a0 +; ZVFHMIN-NEXT: .cfi_def_cfa sp, 16 +; ZVFHMIN-NEXT: addi sp, sp, 16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv32bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: addi sp, sp, -16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 16 +; ZVFBFA-NEXT: csrr a1, vlenb +; ZVFBFA-NEXT: slli a1, a1, 3 +; ZVFBFA-NEXT: sub sp, sp, a1 +; ZVFBFA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFBFA-NEXT: csrr a2, vlenb +; ZVFBFA-NEXT: vsetvli a1, zero, e8, m4, ta, ma +; ZVFBFA-NEXT: vmset.m v24 +; ZVFBFA-NEXT: slli a1, a2, 1 +; ZVFBFA-NEXT: srli a2, a2, 2 +; ZVFBFA-NEXT: sub a3, a0, a1 +; ZVFBFA-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFBFA-NEXT: vslidedown.vx v0, v24, a2 +; ZVFBFA-NEXT: sltu a2, a0, a3 +; ZVFBFA-NEXT: addi a2, a2, -1 +; ZVFBFA-NEXT: and a2, a2, a3 +; ZVFBFA-NEXT: addi a3, sp, 16 +; ZVFBFA-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: vsetvli zero, a2, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v20, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v12, v16, v0.t +; ZVFBFA-NEXT: bltu a0, a1, .LBB23_2 +; ZVFBFA-NEXT: # %bb.1: +; ZVFBFA-NEXT: mv a0, a1 +; ZVFBFA-NEXT: .LBB23_2: +; ZVFBFA-NEXT: addi a1, sp, 16 +; ZVFBFA-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v24 +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16 +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: add sp, sp, a0 +; ZVFBFA-NEXT: .cfi_def_cfa sp, 16 +; ZVFBFA-NEXT: addi sp, sp, 16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 0 +; ZVFBFA-NEXT: ret %v = call <vscale x 32 x bfloat> @llvm.vp.fadd.nxv32bf16(<vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %b, <vscale x 32 x i1> splat (i1 true), i32 %evl) ret <vscale x 32 x bfloat> %v } define <vscale x 32 x bfloat> @vfadd_vf_nxv32bf16(<vscale x 32 x bfloat> %va, bfloat %b, <vscale x 32 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv32bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: addi sp, sp, -16 -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: slli a1, a1, 4 -; CHECK-NEXT: sub sp, sp, a1 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb -; CHECK-NEXT: vsetvli a1, zero, e16, m8, ta, ma -; CHECK-NEXT: vmv1r.v v7, v0 -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: csrr a2, vlenb -; CHECK-NEXT: vmv.v.x v24, a1 -; CHECK-NEXT: slli a1, a2, 1 -; CHECK-NEXT: srli a2, a2, 2 -; CHECK-NEXT: sub a3, a0, a1 -; CHECK-NEXT: vsetvli a4, zero, e8, mf2, ta, ma -; CHECK-NEXT: vslidedown.vx v0, v0, a2 -; CHECK-NEXT: sltu a2, a0, a3 -; CHECK-NEXT: addi a2, a2, -1 -; CHECK-NEXT: and a2, a2, a3 -; CHECK-NEXT: csrr a3, vlenb -; CHECK-NEXT: slli a3, a3, 3 -; CHECK-NEXT: add a3, sp, a3 -; CHECK-NEXT: addi a3, a3, 16 -; CHECK-NEXT: vs8r.v v24, (a3) # vscale x 64-byte Folded Spill -; CHECK-NEXT: vsetvli zero, a2, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v28, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v12, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v24, v16, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t -; CHECK-NEXT: bltu a0, a1, .LBB24_2 -; CHECK-NEXT: # %bb.1: -; CHECK-NEXT: mv a0, a1 -; CHECK-NEXT: .LBB24_2: -; CHECK-NEXT: vmv1r.v v0, v7 -; CHECK-NEXT: vsetvli zero, a0, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v8, v0.t -; CHECK-NEXT: addi a0, sp, 16 -; CHECK-NEXT: vs8r.v v16, (a0) # vscale x 64-byte Folded Spill -; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: add a0, sp, a0 -; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v16, v0.t -; CHECK-NEXT: addi a0, sp, 16 -; CHECK-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v16, v24, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t -; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 4 -; CHECK-NEXT: add sp, sp, a0 -; CHECK-NEXT: .cfi_def_cfa sp, 16 -; CHECK-NEXT: addi sp, sp, 16 -; CHECK-NEXT: .cfi_def_cfa_offset 0 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv32bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: addi sp, sp, -16 +; ZVFH-NEXT: .cfi_def_cfa_offset 16 +; ZVFH-NEXT: csrr a1, vlenb +; ZVFH-NEXT: slli a1, a1, 4 +; ZVFH-NEXT: sub sp, sp, a1 +; ZVFH-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb +; ZVFH-NEXT: vsetvli a1, zero, e16, m8, ta, ma +; ZVFH-NEXT: vmv1r.v v7, v0 +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: csrr a2, vlenb +; ZVFH-NEXT: vmv.v.x v24, a1 +; ZVFH-NEXT: slli a1, a2, 1 +; ZVFH-NEXT: srli a2, a2, 2 +; ZVFH-NEXT: sub a3, a0, a1 +; ZVFH-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFH-NEXT: vslidedown.vx v0, v0, a2 +; ZVFH-NEXT: sltu a2, a0, a3 +; ZVFH-NEXT: addi a2, a2, -1 +; ZVFH-NEXT: and a2, a2, a3 +; ZVFH-NEXT: csrr a3, vlenb +; ZVFH-NEXT: slli a3, a3, 3 +; ZVFH-NEXT: add a3, sp, a3 +; ZVFH-NEXT: addi a3, a3, 16 +; ZVFH-NEXT: vs8r.v v24, (a3) # vscale x 64-byte Folded Spill +; ZVFH-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v28, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v12, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t +; ZVFH-NEXT: bltu a0, a1, .LBB24_2 +; ZVFH-NEXT: # %bb.1: +; ZVFH-NEXT: mv a0, a1 +; ZVFH-NEXT: .LBB24_2: +; ZVFH-NEXT: vmv1r.v v0, v7 +; ZVFH-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v8, v0.t +; ZVFH-NEXT: addi a0, sp, 16 +; ZVFH-NEXT: vs8r.v v16, (a0) # vscale x 64-byte Folded Spill +; ZVFH-NEXT: csrr a0, vlenb +; ZVFH-NEXT: slli a0, a0, 3 +; ZVFH-NEXT: add a0, sp, a0 +; ZVFH-NEXT: addi a0, a0, 16 +; ZVFH-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v16, v0.t +; ZVFH-NEXT: addi a0, sp, 16 +; ZVFH-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; ZVFH-NEXT: csrr a0, vlenb +; ZVFH-NEXT: slli a0, a0, 4 +; ZVFH-NEXT: add sp, sp, a0 +; ZVFH-NEXT: .cfi_def_cfa sp, 16 +; ZVFH-NEXT: addi sp, sp, 16 +; ZVFH-NEXT: .cfi_def_cfa_offset 0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv32bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: addi sp, sp, -16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 16 +; ZVFHMIN-NEXT: csrr a1, vlenb +; ZVFHMIN-NEXT: slli a1, a1, 4 +; ZVFHMIN-NEXT: sub sp, sp, a1 +; ZVFHMIN-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb +; ZVFHMIN-NEXT: vsetvli a1, zero, e16, m8, ta, ma +; ZVFHMIN-NEXT: vmv1r.v v7, v0 +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: csrr a2, vlenb +; ZVFHMIN-NEXT: vmv.v.x v24, a1 +; ZVFHMIN-NEXT: slli a1, a2, 1 +; ZVFHMIN-NEXT: srli a2, a2, 2 +; ZVFHMIN-NEXT: sub a3, a0, a1 +; ZVFHMIN-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFHMIN-NEXT: vslidedown.vx v0, v0, a2 +; ZVFHMIN-NEXT: sltu a2, a0, a3 +; ZVFHMIN-NEXT: addi a2, a2, -1 +; ZVFHMIN-NEXT: and a2, a2, a3 +; ZVFHMIN-NEXT: csrr a3, vlenb +; ZVFHMIN-NEXT: slli a3, a3, 3 +; ZVFHMIN-NEXT: add a3, sp, a3 +; ZVFHMIN-NEXT: addi a3, a3, 16 +; ZVFHMIN-NEXT: vs8r.v v24, (a3) # vscale x 64-byte Folded Spill +; ZVFHMIN-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v28, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v12, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t +; ZVFHMIN-NEXT: bltu a0, a1, .LBB24_2 +; ZVFHMIN-NEXT: # %bb.1: +; ZVFHMIN-NEXT: mv a0, a1 +; ZVFHMIN-NEXT: .LBB24_2: +; ZVFHMIN-NEXT: vmv1r.v v0, v7 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v8, v0.t +; ZVFHMIN-NEXT: addi a0, sp, 16 +; ZVFHMIN-NEXT: vs8r.v v16, (a0) # vscale x 64-byte Folded Spill +; ZVFHMIN-NEXT: csrr a0, vlenb +; ZVFHMIN-NEXT: slli a0, a0, 3 +; ZVFHMIN-NEXT: add a0, sp, a0 +; ZVFHMIN-NEXT: addi a0, a0, 16 +; ZVFHMIN-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v16, v0.t +; ZVFHMIN-NEXT: addi a0, sp, 16 +; ZVFHMIN-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; ZVFHMIN-NEXT: csrr a0, vlenb +; ZVFHMIN-NEXT: slli a0, a0, 4 +; ZVFHMIN-NEXT: add sp, sp, a0 +; ZVFHMIN-NEXT: .cfi_def_cfa sp, 16 +; ZVFHMIN-NEXT: addi sp, sp, 16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv32bf16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: addi sp, sp, -16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 16 +; ZVFBFA-NEXT: csrr a1, vlenb +; ZVFBFA-NEXT: slli a1, a1, 4 +; ZVFBFA-NEXT: sub sp, sp, a1 +; ZVFBFA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb +; ZVFBFA-NEXT: vsetvli a1, zero, e16, m8, ta, ma +; ZVFBFA-NEXT: vmv1r.v v7, v0 +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: csrr a2, vlenb +; ZVFBFA-NEXT: vmv.v.x v24, a1 +; ZVFBFA-NEXT: slli a1, a2, 1 +; ZVFBFA-NEXT: srli a2, a2, 2 +; ZVFBFA-NEXT: sub a3, a0, a1 +; ZVFBFA-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFBFA-NEXT: vslidedown.vx v0, v0, a2 +; ZVFBFA-NEXT: sltu a2, a0, a3 +; ZVFBFA-NEXT: addi a2, a2, -1 +; ZVFBFA-NEXT: and a2, a2, a3 +; ZVFBFA-NEXT: csrr a3, vlenb +; ZVFBFA-NEXT: slli a3, a3, 3 +; ZVFBFA-NEXT: add a3, sp, a3 +; ZVFBFA-NEXT: addi a3, a3, 16 +; ZVFBFA-NEXT: vs8r.v v24, (a3) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: vsetvli zero, a2, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v28, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v12, v16, v0.t +; ZVFBFA-NEXT: bltu a0, a1, .LBB24_2 +; ZVFBFA-NEXT: # %bb.1: +; ZVFBFA-NEXT: mv a0, a1 +; ZVFBFA-NEXT: .LBB24_2: +; ZVFBFA-NEXT: vmv1r.v v0, v7 +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8, v0.t +; ZVFBFA-NEXT: addi a0, sp, 16 +; ZVFBFA-NEXT: vs8r.v v16, (a0) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: add a0, sp, a0 +; ZVFBFA-NEXT: addi a0, a0, 16 +; ZVFBFA-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v16, v0.t +; ZVFBFA-NEXT: addi a0, sp, 16 +; ZVFBFA-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16, v0.t +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 4 +; ZVFBFA-NEXT: add sp, sp, a0 +; ZVFBFA-NEXT: .cfi_def_cfa sp, 16 +; ZVFBFA-NEXT: addi sp, sp, 16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 0 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 32 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 32 x bfloat> %elt.head, <vscale x 32 x bfloat> poison, <vscale x 32 x i32> zeroinitializer %v = call <vscale x 32 x bfloat> @llvm.vp.fadd.nxv32bf16(<vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %vb, <vscale x 32 x i1> %m, i32 %evl) @@ -577,56 +1432,158 @@ define <vscale x 32 x bfloat> @vfadd_vf_nxv32bf16(<vscale x 32 x bfloat> %va, bf } define <vscale x 32 x bfloat> @vfadd_vf_nxv32bf16_unmasked(<vscale x 32 x bfloat> %va, bfloat %b, i32 zeroext %evl) { -; CHECK-LABEL: vfadd_vf_nxv32bf16_unmasked: -; CHECK: # %bb.0: -; CHECK-NEXT: addi sp, sp, -16 -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: slli a1, a1, 3 -; CHECK-NEXT: sub sp, sp, a1 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb -; CHECK-NEXT: fmv.x.h a1, fa0 -; CHECK-NEXT: csrr a2, vlenb -; CHECK-NEXT: vsetvli a3, zero, e16, m8, ta, ma -; CHECK-NEXT: vmset.m v24 -; CHECK-NEXT: vmv.v.x v16, a1 -; CHECK-NEXT: slli a1, a2, 1 -; CHECK-NEXT: srli a2, a2, 2 -; CHECK-NEXT: sub a3, a0, a1 -; CHECK-NEXT: vsetvli a4, zero, e8, mf2, ta, ma -; CHECK-NEXT: vslidedown.vx v0, v24, a2 -; CHECK-NEXT: sltu a2, a0, a3 -; CHECK-NEXT: addi a2, a2, -1 -; CHECK-NEXT: and a2, a2, a3 -; CHECK-NEXT: addi a3, sp, 16 -; CHECK-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill -; CHECK-NEXT: vsetvli zero, a2, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v20, v0.t -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v16, v24, v0.t -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t -; CHECK-NEXT: bltu a0, a1, .LBB25_2 -; CHECK-NEXT: # %bb.1: -; CHECK-NEXT: mv a0, a1 -; CHECK-NEXT: .LBB25_2: -; CHECK-NEXT: vsetvli zero, a0, e16, m4, ta, ma -; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v8 -; CHECK-NEXT: addi a0, sp, 16 -; CHECK-NEXT: vl8r.v v0, (a0) # vscale x 64-byte Folded Reload -; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v0 -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfadd.vv v16, v16, v24 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vfncvtbf16.f.f.w v8, v16 -; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: add sp, sp, a0 -; CHECK-NEXT: .cfi_def_cfa sp, 16 -; CHECK-NEXT: addi sp, sp, 16 -; CHECK-NEXT: .cfi_def_cfa_offset 0 -; CHECK-NEXT: ret +; ZVFH-LABEL: vfadd_vf_nxv32bf16_unmasked: +; ZVFH: # %bb.0: +; ZVFH-NEXT: addi sp, sp, -16 +; ZVFH-NEXT: .cfi_def_cfa_offset 16 +; ZVFH-NEXT: csrr a1, vlenb +; ZVFH-NEXT: slli a1, a1, 3 +; ZVFH-NEXT: sub sp, sp, a1 +; ZVFH-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFH-NEXT: fmv.x.h a1, fa0 +; ZVFH-NEXT: csrr a2, vlenb +; ZVFH-NEXT: vsetvli a3, zero, e16, m8, ta, ma +; ZVFH-NEXT: vmset.m v24 +; ZVFH-NEXT: vmv.v.x v16, a1 +; ZVFH-NEXT: slli a1, a2, 1 +; ZVFH-NEXT: srli a2, a2, 2 +; ZVFH-NEXT: sub a3, a0, a1 +; ZVFH-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFH-NEXT: vslidedown.vx v0, v24, a2 +; ZVFH-NEXT: sltu a2, a0, a3 +; ZVFH-NEXT: addi a2, a2, -1 +; ZVFH-NEXT: and a2, a2, a3 +; ZVFH-NEXT: addi a3, sp, 16 +; ZVFH-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFH-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v20, v0.t +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t +; ZVFH-NEXT: bltu a0, a1, .LBB25_2 +; ZVFH-NEXT: # %bb.1: +; ZVFH-NEXT: mv a0, a1 +; ZVFH-NEXT: .LBB25_2: +; ZVFH-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFH-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFH-NEXT: addi a0, sp, 16 +; ZVFH-NEXT: vl8r.v v0, (a0) # vscale x 64-byte Folded Reload +; ZVFH-NEXT: vfwcvtbf16.f.f.v v24, v0 +; ZVFH-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFH-NEXT: vfadd.vv v16, v16, v24 +; ZVFH-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFH-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFH-NEXT: csrr a0, vlenb +; ZVFH-NEXT: slli a0, a0, 3 +; ZVFH-NEXT: add sp, sp, a0 +; ZVFH-NEXT: .cfi_def_cfa sp, 16 +; ZVFH-NEXT: addi sp, sp, 16 +; ZVFH-NEXT: .cfi_def_cfa_offset 0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vfadd_vf_nxv32bf16_unmasked: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: addi sp, sp, -16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 16 +; ZVFHMIN-NEXT: csrr a1, vlenb +; ZVFHMIN-NEXT: slli a1, a1, 3 +; ZVFHMIN-NEXT: sub sp, sp, a1 +; ZVFHMIN-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFHMIN-NEXT: fmv.x.h a1, fa0 +; ZVFHMIN-NEXT: csrr a2, vlenb +; ZVFHMIN-NEXT: vsetvli a3, zero, e16, m8, ta, ma +; ZVFHMIN-NEXT: vmset.m v24 +; ZVFHMIN-NEXT: vmv.v.x v16, a1 +; ZVFHMIN-NEXT: slli a1, a2, 1 +; ZVFHMIN-NEXT: srli a2, a2, 2 +; ZVFHMIN-NEXT: sub a3, a0, a1 +; ZVFHMIN-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFHMIN-NEXT: vslidedown.vx v0, v24, a2 +; ZVFHMIN-NEXT: sltu a2, a0, a3 +; ZVFHMIN-NEXT: addi a2, a2, -1 +; ZVFHMIN-NEXT: and a2, a2, a3 +; ZVFHMIN-NEXT: addi a3, sp, 16 +; ZVFHMIN-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFHMIN-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v20, v0.t +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v12, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v12, v16, v0.t +; ZVFHMIN-NEXT: bltu a0, a1, .LBB25_2 +; ZVFHMIN-NEXT: # %bb.1: +; ZVFHMIN-NEXT: mv a0, a1 +; ZVFHMIN-NEXT: .LBB25_2: +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v16, v8 +; ZVFHMIN-NEXT: addi a0, sp, 16 +; ZVFHMIN-NEXT: vl8r.v v0, (a0) # vscale x 64-byte Folded Reload +; ZVFHMIN-NEXT: vfwcvtbf16.f.f.v v24, v0 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfadd.vv v16, v16, v24 +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16 +; ZVFHMIN-NEXT: csrr a0, vlenb +; ZVFHMIN-NEXT: slli a0, a0, 3 +; ZVFHMIN-NEXT: add sp, sp, a0 +; ZVFHMIN-NEXT: .cfi_def_cfa sp, 16 +; ZVFHMIN-NEXT: addi sp, sp, 16 +; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 +; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv32bf16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: addi sp, sp, -16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 16 +; ZVFBFA-NEXT: csrr a1, vlenb +; ZVFBFA-NEXT: slli a1, a1, 3 +; ZVFBFA-NEXT: sub sp, sp, a1 +; ZVFBFA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFBFA-NEXT: fmv.x.h a1, fa0 +; ZVFBFA-NEXT: csrr a2, vlenb +; ZVFBFA-NEXT: vsetvli a3, zero, e16, m8, ta, ma +; ZVFBFA-NEXT: vmset.m v24 +; ZVFBFA-NEXT: vmv.v.x v16, a1 +; ZVFBFA-NEXT: slli a1, a2, 1 +; ZVFBFA-NEXT: srli a2, a2, 2 +; ZVFBFA-NEXT: sub a3, a0, a1 +; ZVFBFA-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFBFA-NEXT: vslidedown.vx v0, v24, a2 +; ZVFBFA-NEXT: sltu a2, a0, a3 +; ZVFBFA-NEXT: addi a2, a2, -1 +; ZVFBFA-NEXT: and a2, a2, a3 +; ZVFBFA-NEXT: addi a3, sp, 16 +; ZVFBFA-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: vsetvli zero, a2, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v20, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v12, v16, v0.t +; ZVFBFA-NEXT: bltu a0, a1, .LBB25_2 +; ZVFBFA-NEXT: # %bb.1: +; ZVFBFA-NEXT: mv a0, a1 +; ZVFBFA-NEXT: .LBB25_2: +; ZVFBFA-NEXT: vsetvli zero, a0, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8 +; ZVFBFA-NEXT: addi a0, sp, 16 +; ZVFBFA-NEXT: vl8r.v v0, (a0) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v0 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v24 +; ZVFBFA-NEXT: vsetvli zero, zero, e16alt, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16 +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: add sp, sp, a0 +; ZVFBFA-NEXT: .cfi_def_cfa sp, 16 +; ZVFBFA-NEXT: addi sp, sp, 16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 0 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 32 x bfloat> poison, bfloat %b, i32 0 %vb = shufflevector <vscale x 32 x bfloat> %elt.head, <vscale x 32 x bfloat> poison, <vscale x 32 x i32> zeroinitializer %v = call <vscale x 32 x bfloat> @llvm.vp.fadd.nxv32bf16(<vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %vb, <vscale x 32 x i1> splat (i1 true), i32 %evl) @@ -651,6 +1608,17 @@ define <vscale x 1 x half> @vfadd_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9, v0.t ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv1f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v9, v10, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9, v0.t +; ZVFBFA-NEXT: ret %v = call <vscale x 1 x half> @llvm.vp.fadd.nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x half> %b, <vscale x 1 x i1> %m, i32 %evl) ret <vscale x 1 x half> %v } @@ -672,6 +1640,17 @@ define <vscale x 1 x half> @vfadd_vv_nxv1f16_unmasked(<vscale x 1 x half> %va, < ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv1f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9 +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v9, v10 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %v = call <vscale x 1 x half> @llvm.vp.fadd.nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x half> %b, <vscale x 1 x i1> splat (i1 true), i32 %evl) ret <vscale x 1 x half> %v } @@ -695,6 +1674,19 @@ define <vscale x 1 x half> @vfadd_vf_nxv1f16(<vscale x 1 x half> %va, half %b, < ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9, v0.t ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv1f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v10, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer %v = call <vscale x 1 x half> @llvm.vp.fadd.nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x half> %vb, <vscale x 1 x i1> %m, i32 %evl) @@ -720,6 +1712,19 @@ define <vscale x 1 x half> @vfadd_vf_nxv1f16_commute(<vscale x 1 x half> %va, ha ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9, v0.t ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv1f16_commute: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v8, v10, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer %v = call <vscale x 1 x half> @llvm.vp.fadd.nxv1f16(<vscale x 1 x half> %vb, <vscale x 1 x half> %va, <vscale x 1 x i1> %m, i32 %evl) @@ -745,6 +1750,19 @@ define <vscale x 1 x half> @vfadd_vf_nxv1f16_unmasked(<vscale x 1 x half> %va, h ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv1f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v10, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer %v = call <vscale x 1 x half> @llvm.vp.fadd.nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x half> %vb, <vscale x 1 x i1> splat (i1 true), i32 %evl) @@ -770,6 +1788,19 @@ define <vscale x 1 x half> @vfadd_vf_nxv1f16_unmasked_commute(<vscale x 1 x half ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv1f16_unmasked_commute: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v8, v10 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer %v = call <vscale x 1 x half> @llvm.vp.fadd.nxv1f16(<vscale x 1 x half> %vb, <vscale x 1 x half> %va, <vscale x 1 x i1> splat (i1 true), i32 %evl) @@ -795,6 +1826,17 @@ define <vscale x 2 x half> @vfadd_vv_nxv2f16(<vscale x 2 x half> %va, <vscale x ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9, v0.t ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv2f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v9, v10, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9, v0.t +; ZVFBFA-NEXT: ret %v = call <vscale x 2 x half> @llvm.vp.fadd.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %b, <vscale x 2 x i1> %m, i32 %evl) ret <vscale x 2 x half> %v } @@ -816,6 +1858,17 @@ define <vscale x 2 x half> @vfadd_vv_nxv2f16_unmasked(<vscale x 2 x half> %va, < ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv2f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9 +; ZVFBFA-NEXT: vfwcvt.f.f.v v9, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v9, v10 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %v = call <vscale x 2 x half> @llvm.vp.fadd.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %b, <vscale x 2 x i1> splat (i1 true), i32 %evl) ret <vscale x 2 x half> %v } @@ -839,6 +1892,19 @@ define <vscale x 2 x half> @vfadd_vf_nxv2f16(<vscale x 2 x half> %va, half %b, < ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9, v0.t ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv2f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v10, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 2 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 2 x half> %elt.head, <vscale x 2 x half> poison, <vscale x 2 x i32> zeroinitializer %v = call <vscale x 2 x half> @llvm.vp.fadd.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, <vscale x 2 x i1> %m, i32 %evl) @@ -864,6 +1930,19 @@ define <vscale x 2 x half> @vfadd_vf_nxv2f16_unmasked(<vscale x 2 x half> %va, h ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v9 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv2f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; ZVFBFA-NEXT: vmv.v.x v9, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v9 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; ZVFBFA-NEXT: vfadd.vv v9, v10, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v9 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 2 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 2 x half> %elt.head, <vscale x 2 x half> poison, <vscale x 2 x i32> zeroinitializer %v = call <vscale x 2 x half> @llvm.vp.fadd.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, <vscale x 2 x i1> splat (i1 true), i32 %evl) @@ -889,6 +1968,17 @@ define <vscale x 4 x half> @vfadd_vv_nxv4f16(<vscale x 4 x half> %va, <vscale x ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v10, v0.t ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv4f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v10, v12, v10, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v10, v0.t +; ZVFBFA-NEXT: ret %v = call <vscale x 4 x half> @llvm.vp.fadd.nxv4f16(<vscale x 4 x half> %va, <vscale x 4 x half> %b, <vscale x 4 x i1> %m, i32 %evl) ret <vscale x 4 x half> %v } @@ -910,6 +2000,17 @@ define <vscale x 4 x half> @vfadd_vv_nxv4f16_unmasked(<vscale x 4 x half> %va, < ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v10 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv4f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v9 +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v10, v12, v10 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v10 +; ZVFBFA-NEXT: ret %v = call <vscale x 4 x half> @llvm.vp.fadd.nxv4f16(<vscale x 4 x half> %va, <vscale x 4 x half> %b, <vscale x 4 x i1> splat (i1 true), i32 %evl) ret <vscale x 4 x half> %v } @@ -933,6 +2034,19 @@ define <vscale x 4 x half> @vfadd_vf_nxv4f16(<vscale x 4 x half> %va, half %b, < ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v10, v0.t ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv4f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFBFA-NEXT: vmv.v.x v12, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v10, v10, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v10, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 4 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 4 x half> %elt.head, <vscale x 4 x half> poison, <vscale x 4 x i32> zeroinitializer %v = call <vscale x 4 x half> @llvm.vp.fadd.nxv4f16(<vscale x 4 x half> %va, <vscale x 4 x half> %vb, <vscale x 4 x i1> %m, i32 %evl) @@ -958,6 +2072,19 @@ define <vscale x 4 x half> @vfadd_vf_nxv4f16_unmasked(<vscale x 4 x half> %va, h ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v10 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv4f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; ZVFBFA-NEXT: vmv.v.x v12, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v10, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v12 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; ZVFBFA-NEXT: vfadd.vv v10, v10, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v10 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 4 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 4 x half> %elt.head, <vscale x 4 x half> poison, <vscale x 4 x i32> zeroinitializer %v = call <vscale x 4 x half> @llvm.vp.fadd.nxv4f16(<vscale x 4 x half> %va, <vscale x 4 x half> %vb, <vscale x 4 x i1> splat (i1 true), i32 %evl) @@ -983,6 +2110,17 @@ define <vscale x 8 x half> @vfadd_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v12, v0.t ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv8f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v10, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vv v12, v16, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v12, v0.t +; ZVFBFA-NEXT: ret %v = call <vscale x 8 x half> @llvm.vp.fadd.nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x half> %b, <vscale x 8 x i1> %m, i32 %evl) ret <vscale x 8 x half> %v } @@ -1004,6 +2142,17 @@ define <vscale x 8 x half> @vfadd_vv_nxv8f16_unmasked(<vscale x 8 x half> %va, < ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v12 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv8f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v10 +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vv v12, v16, v12 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v12 +; ZVFBFA-NEXT: ret %v = call <vscale x 8 x half> @llvm.vp.fadd.nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x half> %b, <vscale x 8 x i1> splat (i1 true), i32 %evl) ret <vscale x 8 x half> %v } @@ -1027,6 +2176,19 @@ define <vscale x 8 x half> @vfadd_vf_nxv8f16(<vscale x 8 x half> %va, half %b, < ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v12, v0.t ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv8f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFBFA-NEXT: vmv.v.x v16, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v16, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vv v12, v12, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v12, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 8 x half> %elt.head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer %v = call <vscale x 8 x half> @llvm.vp.fadd.nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x half> %vb, <vscale x 8 x i1> %m, i32 %evl) @@ -1052,6 +2214,19 @@ define <vscale x 8 x half> @vfadd_vf_nxv8f16_unmasked(<vscale x 8 x half> %va, h ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v12 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv8f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; ZVFBFA-NEXT: vmv.v.x v16, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v12, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v16 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; ZVFBFA-NEXT: vfadd.vv v12, v12, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v12 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 8 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 8 x half> %elt.head, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer %v = call <vscale x 8 x half> @llvm.vp.fadd.nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x half> %vb, <vscale x 8 x i1> splat (i1 true), i32 %evl) @@ -1077,6 +2252,17 @@ define <vscale x 16 x half> @vfadd_vv_nxv16f16(<vscale x 16 x half> %va, <vscale ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v16, v0.t ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv16f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16, v0.t +; ZVFBFA-NEXT: ret %v = call <vscale x 16 x half> @llvm.vp.fadd.nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x half> %b, <vscale x 16 x i1> %m, i32 %evl) ret <vscale x 16 x half> %v } @@ -1098,6 +2284,17 @@ define <vscale x 16 x half> @vfadd_vv_nxv16f16_unmasked(<vscale x 16 x half> %va ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v16 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv16f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12 +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16 +; ZVFBFA-NEXT: ret %v = call <vscale x 16 x half> @llvm.vp.fadd.nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x half> %b, <vscale x 16 x i1> splat (i1 true), i32 %evl) ret <vscale x 16 x half> %v } @@ -1121,6 +2318,19 @@ define <vscale x 16 x half> @vfadd_vf_nxv16f16(<vscale x 16 x half> %va, half %b ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v16, v0.t ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv16f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v24, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v24, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16, v0.t +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 16 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 16 x half> %elt.head, <vscale x 16 x half> poison, <vscale x 16 x i32> zeroinitializer %v = call <vscale x 16 x half> @llvm.vp.fadd.nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x half> %vb, <vscale x 16 x i1> %m, i32 %evl) @@ -1146,6 +2356,19 @@ define <vscale x 16 x half> @vfadd_vf_nxv16f16_unmasked(<vscale x 16 x half> %va ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v16 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv16f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFBFA-NEXT: vmv.v.x v24, a1 +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8 +; ZVFBFA-NEXT: vfwcvt.f.f.v v8, v24 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 16 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 16 x half> %elt.head, <vscale x 16 x half> poison, <vscale x 16 x i32> zeroinitializer %v = call <vscale x 16 x half> @llvm.vp.fadd.nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x half> %vb, <vscale x 16 x i1> splat (i1 true), i32 %evl) @@ -1209,6 +2432,55 @@ define <vscale x 32 x half> @vfadd_vv_nxv32f16(<vscale x 32 x half> %va, <vscale ; ZVFHMIN-NEXT: addi sp, sp, 16 ; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv32f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: addi sp, sp, -16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 16 +; ZVFBFA-NEXT: csrr a1, vlenb +; ZVFBFA-NEXT: slli a1, a1, 3 +; ZVFBFA-NEXT: sub sp, sp, a1 +; ZVFBFA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFBFA-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; ZVFBFA-NEXT: vmv1r.v v7, v0 +; ZVFBFA-NEXT: csrr a2, vlenb +; ZVFBFA-NEXT: slli a1, a2, 1 +; ZVFBFA-NEXT: srli a2, a2, 2 +; ZVFBFA-NEXT: sub a3, a0, a1 +; ZVFBFA-NEXT: vslidedown.vx v0, v0, a2 +; ZVFBFA-NEXT: sltu a2, a0, a3 +; ZVFBFA-NEXT: addi a2, a2, -1 +; ZVFBFA-NEXT: and a2, a2, a3 +; ZVFBFA-NEXT: addi a3, sp, 16 +; ZVFBFA-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v20, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v12, v16, v0.t +; ZVFBFA-NEXT: bltu a0, a1, .LBB48_2 +; ZVFBFA-NEXT: # %bb.1: +; ZVFBFA-NEXT: mv a0, a1 +; ZVFBFA-NEXT: .LBB48_2: +; ZVFBFA-NEXT: vmv1r.v v0, v7 +; ZVFBFA-NEXT: addi a1, sp, 16 +; ZVFBFA-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v24, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v8, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16, v0.t +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: add sp, sp, a0 +; ZVFBFA-NEXT: .cfi_def_cfa sp, 16 +; ZVFBFA-NEXT: addi sp, sp, 16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 0 +; ZVFBFA-NEXT: ret %v = call <vscale x 32 x half> @llvm.vp.fadd.nxv32f16(<vscale x 32 x half> %va, <vscale x 32 x half> %b, <vscale x 32 x i1> %m, i32 %evl) ret <vscale x 32 x half> %v } @@ -1268,6 +2540,55 @@ define <vscale x 32 x half> @vfadd_vv_nxv32f16_unmasked(<vscale x 32 x half> %va ; ZVFHMIN-NEXT: addi sp, sp, 16 ; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vv_nxv32f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: addi sp, sp, -16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 16 +; ZVFBFA-NEXT: csrr a1, vlenb +; ZVFBFA-NEXT: slli a1, a1, 3 +; ZVFBFA-NEXT: sub sp, sp, a1 +; ZVFBFA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFBFA-NEXT: csrr a2, vlenb +; ZVFBFA-NEXT: vsetvli a1, zero, e8, m4, ta, ma +; ZVFBFA-NEXT: vmset.m v24 +; ZVFBFA-NEXT: slli a1, a2, 1 +; ZVFBFA-NEXT: srli a2, a2, 2 +; ZVFBFA-NEXT: sub a3, a0, a1 +; ZVFBFA-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFBFA-NEXT: vslidedown.vx v0, v24, a2 +; ZVFBFA-NEXT: sltu a2, a0, a3 +; ZVFBFA-NEXT: addi a2, a2, -1 +; ZVFBFA-NEXT: and a2, a2, a3 +; ZVFBFA-NEXT: addi a3, sp, 16 +; ZVFBFA-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v20, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v12, v16, v0.t +; ZVFBFA-NEXT: bltu a0, a1, .LBB49_2 +; ZVFBFA-NEXT: # %bb.1: +; ZVFBFA-NEXT: mv a0, a1 +; ZVFBFA-NEXT: .LBB49_2: +; ZVFBFA-NEXT: addi a1, sp, 16 +; ZVFBFA-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v24 +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v8 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16 +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: add sp, sp, a0 +; ZVFBFA-NEXT: .cfi_def_cfa sp, 16 +; ZVFBFA-NEXT: addi sp, sp, 16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 0 +; ZVFBFA-NEXT: ret %v = call <vscale x 32 x half> @llvm.vp.fadd.nxv32f16(<vscale x 32 x half> %va, <vscale x 32 x half> %b, <vscale x 32 x i1> splat (i1 true), i32 %evl) ret <vscale x 32 x half> %v } @@ -1340,6 +2661,68 @@ define <vscale x 32 x half> @vfadd_vf_nxv32f16(<vscale x 32 x half> %va, half %b ; ZVFHMIN-NEXT: addi sp, sp, 16 ; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv32f16: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: addi sp, sp, -16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 16 +; ZVFBFA-NEXT: csrr a1, vlenb +; ZVFBFA-NEXT: slli a1, a1, 4 +; ZVFBFA-NEXT: sub sp, sp, a1 +; ZVFBFA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb +; ZVFBFA-NEXT: vsetvli a1, zero, e16, m8, ta, ma +; ZVFBFA-NEXT: vmv1r.v v7, v0 +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: csrr a2, vlenb +; ZVFBFA-NEXT: vmv.v.x v24, a1 +; ZVFBFA-NEXT: slli a1, a2, 1 +; ZVFBFA-NEXT: srli a2, a2, 2 +; ZVFBFA-NEXT: sub a3, a0, a1 +; ZVFBFA-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFBFA-NEXT: vslidedown.vx v0, v0, a2 +; ZVFBFA-NEXT: sltu a2, a0, a3 +; ZVFBFA-NEXT: addi a2, a2, -1 +; ZVFBFA-NEXT: and a2, a2, a3 +; ZVFBFA-NEXT: csrr a3, vlenb +; ZVFBFA-NEXT: slli a3, a3, 3 +; ZVFBFA-NEXT: add a3, sp, a3 +; ZVFBFA-NEXT: addi a3, a3, 16 +; ZVFBFA-NEXT: vs8r.v v24, (a3) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v28, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v24, v16, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v12, v16, v0.t +; ZVFBFA-NEXT: bltu a0, a1, .LBB50_2 +; ZVFBFA-NEXT: # %bb.1: +; ZVFBFA-NEXT: mv a0, a1 +; ZVFBFA-NEXT: .LBB50_2: +; ZVFBFA-NEXT: vmv1r.v v0, v7 +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8, v0.t +; ZVFBFA-NEXT: addi a0, sp, 16 +; ZVFBFA-NEXT: vs8r.v v16, (a0) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: add a0, sp, a0 +; ZVFBFA-NEXT: addi a0, a0, 16 +; ZVFBFA-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v16, v0.t +; ZVFBFA-NEXT: addi a0, sp, 16 +; ZVFBFA-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16, v0.t +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 4 +; ZVFBFA-NEXT: add sp, sp, a0 +; ZVFBFA-NEXT: .cfi_def_cfa sp, 16 +; ZVFBFA-NEXT: addi sp, sp, 16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 0 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 32 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 32 x half> %elt.head, <vscale x 32 x half> poison, <vscale x 32 x i32> zeroinitializer %v = call <vscale x 32 x half> @llvm.vp.fadd.nxv32f16(<vscale x 32 x half> %va, <vscale x 32 x half> %vb, <vscale x 32 x i1> %m, i32 %evl) @@ -1403,6 +2786,57 @@ define <vscale x 32 x half> @vfadd_vf_nxv32f16_unmasked(<vscale x 32 x half> %va ; ZVFHMIN-NEXT: addi sp, sp, 16 ; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 ; ZVFHMIN-NEXT: ret +; +; ZVFBFA-LABEL: vfadd_vf_nxv32f16_unmasked: +; ZVFBFA: # %bb.0: +; ZVFBFA-NEXT: addi sp, sp, -16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 16 +; ZVFBFA-NEXT: csrr a1, vlenb +; ZVFBFA-NEXT: slli a1, a1, 3 +; ZVFBFA-NEXT: sub sp, sp, a1 +; ZVFBFA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb +; ZVFBFA-NEXT: fmv.x.w a1, fa0 +; ZVFBFA-NEXT: csrr a2, vlenb +; ZVFBFA-NEXT: vsetvli a3, zero, e16, m8, ta, ma +; ZVFBFA-NEXT: vmset.m v24 +; ZVFBFA-NEXT: vmv.v.x v16, a1 +; ZVFBFA-NEXT: slli a1, a2, 1 +; ZVFBFA-NEXT: srli a2, a2, 2 +; ZVFBFA-NEXT: sub a3, a0, a1 +; ZVFBFA-NEXT: vsetvli a4, zero, e8, mf2, ta, ma +; ZVFBFA-NEXT: vslidedown.vx v0, v24, a2 +; ZVFBFA-NEXT: sltu a2, a0, a3 +; ZVFBFA-NEXT: addi a2, a2, -1 +; ZVFBFA-NEXT: and a2, a2, a3 +; ZVFBFA-NEXT: addi a3, sp, 16 +; ZVFBFA-NEXT: vs8r.v v16, (a3) # vscale x 64-byte Folded Spill +; ZVFBFA-NEXT: vsetvli zero, a2, e16, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v20, v0.t +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v12, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v24, v0.t +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v12, v16, v0.t +; ZVFBFA-NEXT: bltu a0, a1, .LBB51_2 +; ZVFBFA-NEXT: # %bb.1: +; ZVFBFA-NEXT: mv a0, a1 +; ZVFBFA-NEXT: .LBB51_2: +; ZVFBFA-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; ZVFBFA-NEXT: vfwcvt.f.f.v v16, v8 +; ZVFBFA-NEXT: addi a0, sp, 16 +; ZVFBFA-NEXT: vl8r.v v0, (a0) # vscale x 64-byte Folded Reload +; ZVFBFA-NEXT: vfwcvt.f.f.v v24, v0 +; ZVFBFA-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFBFA-NEXT: vfadd.vv v16, v16, v24 +; ZVFBFA-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; ZVFBFA-NEXT: vfncvt.f.f.w v8, v16 +; ZVFBFA-NEXT: csrr a0, vlenb +; ZVFBFA-NEXT: slli a0, a0, 3 +; ZVFBFA-NEXT: add sp, sp, a0 +; ZVFBFA-NEXT: .cfi_def_cfa sp, 16 +; ZVFBFA-NEXT: addi sp, sp, 16 +; ZVFBFA-NEXT: .cfi_def_cfa_offset 0 +; ZVFBFA-NEXT: ret %elt.head = insertelement <vscale x 32 x half> poison, half %b, i32 0 %vb = shufflevector <vscale x 32 x half> %elt.head, <vscale x 32 x half> poison, <vscale x 32 x i32> zeroinitializer %v = call <vscale x 32 x half> @llvm.vp.fadd.nxv32f16(<vscale x 32 x half> %va, <vscale x 32 x half> %vb, <vscale x 32 x i1> splat (i1 true), i32 %evl) diff --git a/llvm/test/CodeGen/WebAssembly/memory-interleave.ll b/llvm/test/CodeGen/WebAssembly/memory-interleave.ll index 104ec31..5eb49fd 100644 --- a/llvm/test/CodeGen/WebAssembly/memory-interleave.ll +++ b/llvm/test/CodeGen/WebAssembly/memory-interleave.ll @@ -2103,10 +2103,7 @@ for.body: ; preds = %entry, %for.body ; CHECK-LABEL: four_floats_same_op: ; CHECK: loop -; CHECK: v128.load -; CHECK: v128.load -; CHECK: f32x4.mul -; CHECK: v128.store +; CHECK-NOT: v128.load define hidden void @four_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) { entry: %cmp45.not = icmp eq i32 %N, 0 diff --git a/llvm/test/CodeGen/X86/call-graph-section-addrtaken.ll b/llvm/test/CodeGen/X86/call-graph-section-addrtaken.ll index 632d90d..f36baba 100644 --- a/llvm/test/CodeGen/X86/call-graph-section-addrtaken.ll +++ b/llvm/test/CodeGen/X86/call-graph-section-addrtaken.ll @@ -27,7 +27,7 @@ entry: !1 = !{i64 0, !"_ZTSFivE.generalized"} !2 = !{i64 0, !"_ZTSFviE.generalized"} -; CHECK: .section .llvm.callgraph,"o",@progbits,.text +; CHECK: .section .llvm.callgraph,"o",@llvm_call_graph,.text ;; Version ; CHECK-NEXT: .byte 0 ;; Flags -- Potential indirect target so LSB is set to 1. Other bits are 0. diff --git a/llvm/test/CodeGen/X86/call-graph-section-assembly.ll b/llvm/test/CodeGen/X86/call-graph-section-assembly.ll index ed6849a..cdbad66 100644 --- a/llvm/test/CodeGen/X86/call-graph-section-assembly.ll +++ b/llvm/test/CodeGen/X86/call-graph-section-assembly.ll @@ -36,7 +36,7 @@ entry: !4 = !{!5} !5 = !{i64 0, !"_ZTSFPvS_E.generalized"} -; CHECK: .section .llvm.callgraph,"o",@progbits,.text +; CHECK: .section .llvm.callgraph,"o",@llvm_call_graph,.text ;; Version ; CHECK-NEXT: .byte 0 ;; Flags diff --git a/llvm/test/Transforms/ADCE/2016-09-06.ll b/llvm/test/Transforms/ADCE/2016-09-06.ll index 850f412..1329ac6 100644 --- a/llvm/test/Transforms/ADCE/2016-09-06.ll +++ b/llvm/test/Transforms/ADCE/2016-09-06.ll @@ -5,7 +5,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" ; Function Attrs: nounwind uwtable -define i32 @foo(i32, i32, i32) #0 { +define i32 @foo(i32, i32, i32) { %4 = alloca i32, align 4 %5 = alloca i32, align 4 %6 = alloca i32, align 4 @@ -48,8 +48,6 @@ B21: ret i32 %I22 } -attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.ident = !{!0} !0 = !{!"clang version 4.0.0"} diff --git a/llvm/test/Transforms/ADCE/blocks-with-dead-term-nondeterministic.ll b/llvm/test/Transforms/ADCE/blocks-with-dead-term-nondeterministic.ll index 9708be9..5e844b4 100644 --- a/llvm/test/Transforms/ADCE/blocks-with-dead-term-nondeterministic.ll +++ b/llvm/test/Transforms/ADCE/blocks-with-dead-term-nondeterministic.ll @@ -5,7 +5,7 @@ target triple = "x86_64-apple-macosx10.10.0" ; CHECK: uselistorder label %bb16, { 1, 0 } ; Function Attrs: noinline nounwind ssp uwtable -define void @ham(i1 %arg) local_unnamed_addr #0 { +define void @ham(i1 %arg) local_unnamed_addr { bb: br i1 false, label %bb1, label %bb22 @@ -64,8 +64,6 @@ bb22: ; preds = %bb21, %bb ret void } -attributes #0 = { noinline nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.module.flags = !{!0} !0 = !{i32 7, !"PIC Level", i32 2} diff --git a/llvm/test/Transforms/AddDiscriminators/basic.ll b/llvm/test/Transforms/AddDiscriminators/basic.ll index 5186537..fc4c10a 100644 --- a/llvm/test/Transforms/AddDiscriminators/basic.ll +++ b/llvm/test/Transforms/AddDiscriminators/basic.ll @@ -11,7 +11,7 @@ ; if (i < 10) x = i; ; } -define void @foo(i32 %i) #0 !dbg !4 { +define void @foo(i32 %i) !dbg !4 { entry: %i.addr = alloca i32, align 4 %x = alloca i32, align 4 @@ -35,8 +35,6 @@ if.end: ; preds = %if.then, %entry ; CHECK: ret void, !dbg ![[END:[0-9]+]] } -attributes #0 = { nounwind uwtable noinline optnone "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!7, !8} !llvm.ident = !{!9} diff --git a/llvm/test/Transforms/AddDiscriminators/call-nested.ll b/llvm/test/Transforms/AddDiscriminators/call-nested.ll index 99340a5..f1373e4 100644 --- a/llvm/test/Transforms/AddDiscriminators/call-nested.ll +++ b/llvm/test/Transforms/AddDiscriminators/call-nested.ll @@ -9,7 +9,7 @@ ; #6 } ; Function Attrs: uwtable -define i32 @_Z3bazv() #0 !dbg !4 { +define i32 @_Z3bazv() !dbg !4 { %1 = call i32 @_Z3barv(), !dbg !11 ; CHECK: %1 = call i32 @_Z3barv(), !dbg ![[CALL0:[0-9]+]] %2 = call i32 @_Z3barv(), !dbg !12 @@ -19,12 +19,9 @@ define i32 @_Z3bazv() #0 !dbg !4 { ret i32 %3, !dbg !14 } -declare i32 @_Z3fooii(i32, i32) #1 +declare i32 @_Z3fooii(i32, i32) -declare i32 @_Z3barv() #1 - -attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +declare i32 @_Z3barv() !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!8, !9} diff --git a/llvm/test/Transforms/AddDiscriminators/call.ll b/llvm/test/Transforms/AddDiscriminators/call.ll index 93d3aa4..11b21ef 100644 --- a/llvm/test/Transforms/AddDiscriminators/call.ll +++ b/llvm/test/Transforms/AddDiscriminators/call.ll @@ -8,7 +8,7 @@ ; #5 } ; Function Attrs: uwtable -define void @_Z3foov() #0 !dbg !4 { +define void @_Z3foov() !dbg !4 { call void @_Z3barv(), !dbg !10 ; CHECK: call void @_Z3barv(), !dbg ![[CALL0:[0-9]+]] %a = alloca [100 x i8], align 16 @@ -21,13 +21,10 @@ define void @_Z3foov() #0 !dbg !4 { ret void, !dbg !13 } -declare void @_Z3barv() #1 +declare void @_Z3barv() declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind argmemonly declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind argmemonly -attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!7, !8} !llvm.ident = !{!9} diff --git a/llvm/test/Transforms/AddDiscriminators/diamond.ll b/llvm/test/Transforms/AddDiscriminators/diamond.ll index c93a57a..9edcf39 100644 --- a/llvm/test/Transforms/AddDiscriminators/diamond.ll +++ b/llvm/test/Transforms/AddDiscriminators/diamond.ll @@ -12,7 +12,7 @@ ; bar(3): discriminator 2 ; Function Attrs: uwtable -define void @_Z3fooi(i32 %i) #0 !dbg !4 { +define void @_Z3fooi(i32 %i) !dbg !4 { %1 = alloca i32, align 4 store i32 %i, ptr %1, align 4 call void @llvm.dbg.declare(metadata ptr %1, metadata !11, metadata !12), !dbg !13 @@ -34,13 +34,9 @@ define void @_Z3fooi(i32 %i) #0 !dbg !4 { } ; Function Attrs: nounwind readnone -declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 +declare void @llvm.dbg.declare(metadata, metadata, metadata) -declare void @_Z3bari(i32) #2 - -attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind readnone } -attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +declare void @_Z3bari(i32) !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!8, !9} diff --git a/llvm/test/Transforms/AddDiscriminators/first-only.ll b/llvm/test/Transforms/AddDiscriminators/first-only.ll index 7ae9ed0..415e5f0 100644 --- a/llvm/test/Transforms/AddDiscriminators/first-only.ll +++ b/llvm/test/Transforms/AddDiscriminators/first-only.ll @@ -13,7 +13,7 @@ ; } ; } -define void @foo(i32 %i) #0 !dbg !4 { +define void @foo(i32 %i) !dbg !4 { entry: %i.addr = alloca i32, align 4 %x = alloca i32, align 4 @@ -44,8 +44,6 @@ if.end: ; preds = %if.then, %entry ; CHECK: ret void, !dbg ![[END:[0-9]+]] } -attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!7, !8} !llvm.ident = !{!9} diff --git a/llvm/test/Transforms/AddDiscriminators/invoke.ll b/llvm/test/Transforms/AddDiscriminators/invoke.ll index d39014d..a3989b6 100644 --- a/llvm/test/Transforms/AddDiscriminators/invoke.ll +++ b/llvm/test/Transforms/AddDiscriminators/invoke.ll @@ -5,14 +5,14 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.14.0" ; Function Attrs: ssp uwtable -define void @_Z3foov() #0 personality ptr @__gxx_personality_v0 !dbg !8 { +define void @_Z3foov() personality ptr @__gxx_personality_v0 !dbg !8 { entry: %exn.slot = alloca ptr %ehselector.slot = alloca i32 ; CHECK: call void @_Z12bar_noexceptv({{.*}} !dbg ![[CALL1:[0-9]+]] - call void @_Z12bar_noexceptv() #4, !dbg !11 + call void @_Z12bar_noexceptv(), !dbg !11 ; CHECK: call void @_Z12bar_noexceptv({{.*}} !dbg ![[CALL2:[0-9]+]] - call void @_Z12bar_noexceptv() #4, !dbg !13 + call void @_Z12bar_noexceptv(), !dbg !13 invoke void @_Z3barv() ; CHECK: unwind label {{.*}} !dbg ![[INVOKE:[0-9]+]] to label %invoke.cont unwind label %lpad, !dbg !14 @@ -31,8 +31,8 @@ lpad: ; preds = %entry catch: ; preds = %lpad %exn = load ptr, ptr %exn.slot, align 8, !dbg !15 - %3 = call ptr @__cxa_begin_catch(ptr %exn) #4, !dbg !15 - invoke void @__cxa_rethrow() #5 + %3 = call ptr @__cxa_begin_catch(ptr %exn), !dbg !15 + invoke void @__cxa_rethrow() to label %unreachable unwind label %lpad1, !dbg !17 lpad1: ; preds = %catch @@ -62,7 +62,7 @@ terminate.lpad: ; preds = %lpad1 %7 = landingpad { ptr, i32 } catch ptr null, !dbg !20 %8 = extractvalue { ptr, i32 } %7, 0, !dbg !20 - call void @__clang_call_terminate(ptr %8) #6, !dbg !20 + call void @__clang_call_terminate(ptr %8), !dbg !20 unreachable, !dbg !20 unreachable: ; preds = %catch @@ -70,9 +70,9 @@ unreachable: ; preds = %catch } ; Function Attrs: nounwind -declare void @_Z12bar_noexceptv() #1 +declare void @_Z12bar_noexceptv() -declare void @_Z3barv() #2 +declare void @_Z3barv() declare i32 @__gxx_personality_v0(...) @@ -83,22 +83,14 @@ declare void @__cxa_rethrow() declare void @__cxa_end_catch() ; Function Attrs: noinline noreturn nounwind -define linkonce_odr hidden void @__clang_call_terminate(ptr) #3 { - %2 = call ptr @__cxa_begin_catch(ptr %0) #4 - call void @_ZSt9terminatev() #6 +define linkonce_odr hidden void @__clang_call_terminate(ptr) { + %2 = call ptr @__cxa_begin_catch(ptr %0) + call void @_ZSt9terminatev() unreachable } declare void @_ZSt9terminatev() -attributes #0 = { ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #3 = { noinline noreturn nounwind } -attributes #4 = { nounwind } -attributes #5 = { noreturn } -attributes #6 = { noreturn nounwind } - !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5, !6} !llvm.ident = !{!7} diff --git a/llvm/test/Transforms/AddDiscriminators/multiple.ll b/llvm/test/Transforms/AddDiscriminators/multiple.ll index 54c1a5d..8e8ca6a 100644 --- a/llvm/test/Transforms/AddDiscriminators/multiple.ll +++ b/llvm/test/Transforms/AddDiscriminators/multiple.ll @@ -10,7 +10,7 @@ ; The two stores inside the if-then-else line must have different discriminator ; values. -define void @foo(i32 %i) #0 !dbg !4 { +define void @foo(i32 %i) !dbg !4 { entry: %i.addr = alloca i32, align 4 %x = alloca i32, align 4 @@ -45,8 +45,6 @@ if.end: ; preds = %if.else, %if.then ret void, !dbg !12 } -attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!7, !8} !llvm.ident = !{!9} diff --git a/llvm/test/Transforms/AddDiscriminators/no-discriminators.ll b/llvm/test/Transforms/AddDiscriminators/no-discriminators.ll index c23edd6..f84579b 100644 --- a/llvm/test/Transforms/AddDiscriminators/no-discriminators.ll +++ b/llvm/test/Transforms/AddDiscriminators/no-discriminators.ll @@ -12,7 +12,7 @@ ; altered. If they are, it means that the discriminators pass added a ; new lexical scope. -define i32 @foo(i64 %i) #0 !dbg !4 { +define i32 @foo(i64 %i) !dbg !4 { entry: %retval = alloca i32, align 4 %i.addr = alloca i64, align 8 @@ -39,10 +39,7 @@ return: ; preds = %if.else, %if.then } ; Function Attrs: nounwind readnone -declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 - -attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind readnone } +declare void @llvm.dbg.declare(metadata, metadata, metadata) ; We should be able to add discriminators even in the absence of llvm.dbg.cu. ; When using sample profiles, the front end will generate line tables but it diff --git a/llvm/test/Transforms/AddDiscriminators/oneline.ll b/llvm/test/Transforms/AddDiscriminators/oneline.ll index 533d547..fc1675b 100644 --- a/llvm/test/Transforms/AddDiscriminators/oneline.ll +++ b/llvm/test/Transforms/AddDiscriminators/oneline.ll @@ -10,7 +10,7 @@ ; return 100: discriminator 4 ; return 99: discriminator 6 -define i32 @_Z3fooi(i32 %i) #0 !dbg !4 { +define i32 @_Z3fooi(i32 %i) !dbg !4 { %1 = alloca i32, align 4 %2 = alloca i32, align 4 store i32 %i, ptr %2, align 4, !tbaa !13 @@ -49,10 +49,7 @@ define i32 @_Z3fooi(i32 %i) #0 !dbg !4 { } ; Function Attrs: nounwind readnone -declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 - -attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind readnone } +declare void @llvm.dbg.declare(metadata, metadata, metadata) !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!10, !11} diff --git a/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll b/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll index eb7d78f..4704238 100644 --- a/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll +++ b/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll @@ -1557,24 +1557,24 @@ declare dso_local void @_GLOBAL__sub_I_register_benchmark_test.cc() #0 section " ; Function Attrs: cold noreturn nounwind declare void @llvm.trap() #20 -attributes #0 = { uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #2 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #0 = { uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-jump-tables"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } +attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } +attributes #2 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } attributes #3 = { nounwind } -attributes #4 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #4 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-jump-tables"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } attributes #5 = { argmemonly nounwind willreturn } -attributes #6 = { alwaysinline uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #7 = { alwaysinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #8 = { nobuiltin "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #9 = { nobuiltin nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #10 = { inlinehint uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #11 = { inlinehint nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #12 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #13 = { norecurse uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #6 = { alwaysinline uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-jump-tables"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } +attributes #7 = { alwaysinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-jump-tables"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } +attributes #8 = { nobuiltin "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } +attributes #9 = { nobuiltin nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } +attributes #10 = { inlinehint uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-jump-tables"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } +attributes #11 = { inlinehint nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-jump-tables"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } +attributes #12 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } +attributes #13 = { norecurse uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-jump-tables"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } attributes #14 = { nounwind readnone willreturn } -attributes #15 = { noreturn "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #15 = { noreturn "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } attributes #16 = { noinline noreturn nounwind } attributes #17 = { argmemonly nounwind willreturn writeonly } -attributes #18 = { noreturn uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #19 = { inlinehint noreturn uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #18 = { noreturn uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-jump-tables"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } +attributes #19 = { inlinehint noreturn uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-jump-tables"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "use-soft-float"="false" } attributes #20 = { cold noreturn nounwind } diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll index d272fef..189186b 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll @@ -4,7 +4,7 @@ target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "armv7--linux-gnueabihf" ; CHECK-LABEL: @f -define i32 @f(i32 %a) #0 { +define i32 @f(i32 %a) { ; CHECK: call i32 @llvm.bitreverse.i32 entry: br label %for.body @@ -25,8 +25,6 @@ for.body: ; preds = %for.body, %entry br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !3 } -attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.module.flags = !{!0, !1} !llvm.ident = !{!2} diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/bitreverse-hang.ll b/llvm/test/Transforms/CodeGenPrepare/X86/bitreverse-hang.ll index eec0967..35115cf 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/bitreverse-hang.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/bitreverse-hang.ll @@ -21,7 +21,7 @@ @b = common global i32 0, align 4 ; CHECK: define i32 @fn1 -define i32 @fn1() #0 { +define i32 @fn1() { entry: %b.promoted = load i32, ptr @b, align 4, !tbaa !2 br label %for.body @@ -40,8 +40,6 @@ for.end: ; preds = %for.body ret i32 undef } -attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.module.flags = !{!0} !llvm.ident = !{!1} diff --git a/llvm/test/Transforms/CodeGenPrepare/dom-tree.ll b/llvm/test/Transforms/CodeGenPrepare/dom-tree.ll index 1c990ff..14360fe 100644 --- a/llvm/test/Transforms/CodeGenPrepare/dom-tree.ll +++ b/llvm/test/Transforms/CodeGenPrepare/dom-tree.ll @@ -10,7 +10,7 @@ target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "armv7--linux-gnueabihf" -define i32 @f(i32 %a) #0 { +define i32 @f(i32 %a) { entry: br label %for.body @@ -30,8 +30,6 @@ for.body: br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !3 } -attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" } - !llvm.module.flags = !{!0, !1} !llvm.ident = !{!2} diff --git a/llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll b/llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll index 46fa066..78760db 100644 --- a/llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll +++ b/llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll @@ -20,7 +20,7 @@ target triple = "x86_64-pc-windows-msvc" ; BFIHOIST: br label %endif ; Function Attrs: norecurse -define i32 @main(i32 %argc, ptr nocapture readnone %argv) local_unnamed_addr #0 personality ptr @__CxxFrameHandler3 { +define i32 @main(i32 %argc, ptr nocapture readnone %argv) local_unnamed_addr personality ptr @__CxxFrameHandler3 { %call = tail call i64 @fn(i64 0) %call1 = tail call i64 @fn(i64 1) %tobool = icmp eq i32 %argc, 0 @@ -62,9 +62,6 @@ endif: ret i32 0 } -declare i64 @fn(i64) local_unnamed_addr #1 +declare i64 @fn(i64) local_unnamed_addr declare i32 @__CxxFrameHandler3(...) - -attributes #0 = { norecurse "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } diff --git a/llvm/test/Transforms/Coroutines/coro-debug.ll b/llvm/test/Transforms/Coroutines/coro-debug.ll index d1f1922..109be51f 100644 --- a/llvm/test/Transforms/Coroutines/coro-debug.ll +++ b/llvm/test/Transforms/Coroutines/coro-debug.ll @@ -14,7 +14,7 @@ entry: %0 = call token @llvm.coro.id(i32 0, ptr null, ptr @flink, ptr null), !dbg !16 %1 = call i64 @llvm.coro.size.i64(), !dbg !16 %call = call ptr @malloc(i64 %1), !dbg !16 - %2 = call ptr @llvm.coro.begin(token %0, ptr %call) #7, !dbg !16 + %2 = call ptr @llvm.coro.begin(token %0, ptr %call), !dbg !16 store ptr %2, ptr %coro_hdl, align 8, !dbg !16 %3 = call i8 @llvm.coro.suspend(token none, i1 false), !dbg !17 %conv = sext i8 %3 to i32, !dbg !17 @@ -69,7 +69,7 @@ coro_Cleanup: ; preds = %sw.epilog, %sw.bb1 br label %coro_Suspend, !dbg !24 coro_Suspend: ; preds = %coro_Cleanup, %sw.default - call void @llvm.coro.end(ptr null, i1 false, token none) #7, !dbg !24 + call void @llvm.coro.end(ptr null, i1 false, token none), !dbg !24 %7 = load ptr, ptr %coro_hdl, align 8, !dbg !24 store i32 0, ptr %late_local, !dbg !24 ret ptr %7, !dbg !24 @@ -82,47 +82,40 @@ ehcleanup: } ; Function Attrs: nounwind readnone speculatable -declare void @llvm.dbg.value(metadata, metadata, metadata) #1 +declare void @llvm.dbg.value(metadata, metadata, metadata) ; Function Attrs: nounwind readnone speculatable -declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 +declare void @llvm.dbg.declare(metadata, metadata, metadata) ; Function Attrs: argmemonly nounwind readonly -declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #2 +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) -declare ptr @malloc(i64) #3 +declare ptr @malloc(i64) declare ptr @allocate() declare void @print({ ptr, i32 }) declare void @log() ; Function Attrs: nounwind readnone -declare i64 @llvm.coro.size.i64() #4 +declare i64 @llvm.coro.size.i64() ; Function Attrs: nounwind -declare ptr @llvm.coro.begin(token, ptr writeonly) #5 +declare ptr @llvm.coro.begin(token, ptr writeonly) ; Function Attrs: nounwind -declare i8 @llvm.coro.suspend(token, i1) #5 +declare i8 @llvm.coro.suspend(token, i1) -declare void @free(ptr) #3 +declare void @free(ptr) ; Function Attrs: argmemonly nounwind readonly -declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) ; Function Attrs: nounwind -declare void @llvm.coro.end(ptr, i1, token) #5 +declare void @llvm.coro.end(ptr, i1, token) ; Function Attrs: argmemonly nounwind readonly -declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #2 - -attributes #0 = { noinline nounwind presplitcoroutine "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind readnone speculatable } -attributes #2 = { argmemonly nounwind readonly } -attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #4 = { nounwind readnone } -attributes #5 = { nounwind } -attributes #6 = { alwaysinline } -attributes #7 = { noduplicate } +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) + +attributes #0 = { noinline nounwind presplitcoroutine } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} diff --git a/llvm/test/Transforms/Coroutines/coro-split-dbg.ll b/llvm/test/Transforms/Coroutines/coro-split-dbg.ll index c53bea8..577ca9a 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-dbg.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-dbg.ll @@ -6,9 +6,9 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" ; Function Attrs: nounwind readnone -declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 +declare void @llvm.dbg.declare(metadata, metadata, metadata) -declare void @bar(...) local_unnamed_addr #2 +declare void @bar(...) local_unnamed_addr ; Function Attrs: nounwind uwtable define ptr @f() #3 !dbg !16 { @@ -16,14 +16,14 @@ entry: %0 = tail call token @llvm.coro.id(i32 0, ptr null, ptr @f, ptr null), !dbg !26 %1 = tail call i64 @llvm.coro.size.i64(), !dbg !26 %call = tail call ptr @malloc(i64 %1), !dbg !26 - %2 = tail call ptr @llvm.coro.begin(token %0, ptr %call) #9, !dbg !26 + %2 = tail call ptr @llvm.coro.begin(token %0, ptr %call), !dbg !26 tail call void @llvm.dbg.value(metadata ptr %2, metadata !21, metadata !12), !dbg !26 br label %for.cond, !dbg !27 for.cond: ; preds = %for.cond, %entry tail call void @llvm.dbg.value(metadata i32 undef, metadata !22, metadata !12), !dbg !28 - tail call void @llvm.dbg.value(metadata i32 undef, metadata !11, metadata !12) #7, !dbg !29 - tail call void (...) @bar() #7, !dbg !33 + tail call void @llvm.dbg.value(metadata i32 undef, metadata !11, metadata !12), !dbg !29 + tail call void (...) @bar(), !dbg !33 %3 = tail call token @llvm.coro.save(ptr null), !dbg !34 %4 = tail call i8 @llvm.coro.suspend(token %3, i1 false), !dbg !34 %conv = sext i8 %4 to i32, !dbg !34 @@ -38,40 +38,31 @@ coro_Cleanup: ; preds = %for.cond br label %coro_Suspend, !dbg !36 coro_Suspend: ; preds = %for.cond, %if.then, %coro_Cleanup - tail call void @llvm.coro.end(ptr null, i1 false, token none) #9, !dbg !38 + tail call void @llvm.coro.end(ptr null, i1 false, token none), !dbg !38 ret ptr %2, !dbg !39 } ; Function Attrs: argmemonly nounwind -declare void @llvm.lifetime.start.p0(ptr nocapture) #4 +declare void @llvm.lifetime.start.p0(ptr nocapture) ; Function Attrs: argmemonly nounwind readonly -declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #5 +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) ; Function Attrs: nounwind -declare noalias ptr @malloc(i64) local_unnamed_addr #6 -declare i64 @llvm.coro.size.i64() #1 -declare ptr @llvm.coro.begin(token, ptr writeonly) #7 -declare token @llvm.coro.save(ptr) #7 -declare i8 @llvm.coro.suspend(token, i1) #7 -declare void @llvm.lifetime.end.p0(ptr nocapture) #4 -declare ptr @llvm.coro.free(token, ptr nocapture readonly) #5 -declare void @free(ptr nocapture) local_unnamed_addr #6 -declare void @llvm.coro.end(ptr, i1, token) #7 -declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #5 +declare noalias ptr @malloc(i64) local_unnamed_addr +declare i64 @llvm.coro.size.i64() +declare ptr @llvm.coro.begin(token, ptr writeonly) +declare token @llvm.coro.save(ptr) +declare i8 @llvm.coro.suspend(token, i1) +declare void @llvm.lifetime.end.p0(ptr nocapture) +declare ptr @llvm.coro.free(token, ptr nocapture readonly) +declare void @free(ptr nocapture) local_unnamed_addr +declare void @llvm.coro.end(ptr, i1, token) +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) -declare void @llvm.dbg.value(metadata, metadata, metadata) #1 +declare void @llvm.dbg.value(metadata, metadata, metadata) -attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind readnone } -attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #3 = { nounwind uwtable presplitcoroutine "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #4 = { argmemonly nounwind } -attributes #5 = { argmemonly nounwind readonly } -attributes #6 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #7 = { nounwind } -attributes #8 = { alwaysinline nounwind } -attributes #9 = { noduplicate } +attributes #3 = { nounwind uwtable presplitcoroutine } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} diff --git a/llvm/test/Transforms/DeadStoreElimination/matrix-intrinsics.ll b/llvm/test/Transforms/DeadStoreElimination/matrix-intrinsics.ll index ae3c746..2eaa275 100644 --- a/llvm/test/Transforms/DeadStoreElimination/matrix-intrinsics.ll +++ b/llvm/test/Transforms/DeadStoreElimination/matrix-intrinsics.ll @@ -5,8 +5,8 @@ define void @dead_unstrided_store_non_matrix_load(ptr noalias %src, ptr noalias ; CHECK-LABEL: define void @dead_unstrided_store_non_matrix_load( ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: call void @llvm.matrix.column.major.store.v8f64.i32(<8 x double> zeroinitializer, ptr [[DST]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: [[L:%.*]] = load double, ptr [[SRC]], align 8 +; CHECK-NEXT: call void @llvm.matrix.column.major.store.v8f64.i32(<8 x double> zeroinitializer, ptr [[DST]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: ret void ; entry: @@ -173,7 +173,6 @@ define void @dead_unstrided_store(ptr noalias %src, ptr noalias %dst) { ; CHECK-LABEL: define void @dead_unstrided_store( ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: call void @llvm.matrix.column.major.store.v8f64.i32(<8 x double> zeroinitializer, ptr [[DST]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: [[L:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: call void @llvm.matrix.column.major.store.v8f64.i32(<8 x double> [[L]], ptr [[DST]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: ret void @@ -241,7 +240,6 @@ define void @dead_matrix_store_non_matrix_overwrite_unstrided(ptr noalias %src, ; CHECK-LABEL: define void @dead_matrix_store_non_matrix_overwrite_unstrided( ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: call void @llvm.matrix.column.major.store.v8f64.i32(<8 x double> zeroinitializer, ptr [[DST]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: [[L:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: store <8 x double> zeroinitializer, ptr [[DST]], align 64 ; CHECK-NEXT: ret void @@ -257,7 +255,6 @@ define void @dead_matrix_store_non_matrix_overwrite_strided(ptr noalias %src, pt ; CHECK-LABEL: define void @dead_matrix_store_non_matrix_overwrite_strided( ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: call void @llvm.matrix.column.major.store.v8f64.i32(<8 x double> zeroinitializer, ptr [[DST]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: [[L:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC]], i32 8, i1 false, i32 4, i32 2) ; CHECK-NEXT: store <16 x double> zeroinitializer, ptr [[DST]], align 128 ; CHECK-NEXT: ret void @@ -289,7 +286,6 @@ define void @live_matrix_store_non_matrix_overwrite_strided(ptr noalias %src, pt ; CHECK-LABEL: define void @live_matrix_store_non_matrix_overwrite_strided( ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: call void @llvm.matrix.column.major.store.v8f64.i32(<8 x double> zeroinitializer, ptr [[DST]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: [[L:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC]], i32 8, i1 false, i32 4, i32 2) ; CHECK-NEXT: store <8 x double> zeroinitializer, ptr [[DST]], align 64 ; CHECK-NEXT: ret void @@ -305,8 +301,6 @@ define void @dead_matrix_store_dimension_change(ptr noalias %src, ptr noalias %d ; CHECK-LABEL: define void @dead_matrix_store_dimension_change( ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: [[L:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC]], i32 8, i1 false, i32 4, i32 2) -; CHECK-NEXT: call void @llvm.matrix.column.major.store.v8f64.i32(<8 x double> [[L]], ptr [[DST]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: call void @llvm.matrix.column.major.store.v9f64.i32(<9 x double> zeroinitializer, ptr [[DST]], i32 3, i1 false, i32 3, i32 3) ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/GVN/matrix-intrinsics.ll b/llvm/test/Transforms/GVN/matrix-intrinsics.ll index 78dbfe1..03bd45b 100644 --- a/llvm/test/Transforms/GVN/matrix-intrinsics.ll +++ b/llvm/test/Transforms/GVN/matrix-intrinsics.ll @@ -8,9 +8,8 @@ define void @redundant_unstrided_load(ptr %src) { ; CHECK-NEXT: [[SRC_OFFSET:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 8 ; CHECK-NEXT: [[L:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC_OFFSET]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: call void @llvm.matrix.column.major.store.v8f64.i32(<8 x double> [[L]], ptr [[SRC]], i32 4, i1 false, i32 4, i32 2) -; CHECK-NEXT: [[L_2:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC_OFFSET]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: call void @use(<8 x double> [[L]]) -; CHECK-NEXT: call void @use(<8 x double> [[L_2]]) +; CHECK-NEXT: call void @use(<8 x double> [[L]]) ; CHECK-NEXT: ret void ; entry: @@ -30,9 +29,8 @@ define void @redundant_unstrided_load_non_matrix_store(ptr %src) { ; CHECK-NEXT: [[SRC_OFFSET:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 1 ; CHECK-NEXT: [[L:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC_OFFSET]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: store double 4.200000e+01, ptr [[SRC]], align 8 -; CHECK-NEXT: [[L_2:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC_OFFSET]], i32 4, i1 false, i32 4, i32 2) ; CHECK-NEXT: call void @use(<8 x double> [[L]]) -; CHECK-NEXT: call void @use(<8 x double> [[L_2]]) +; CHECK-NEXT: call void @use(<8 x double> [[L]]) ; CHECK-NEXT: ret void ; entry: @@ -52,9 +50,8 @@ define void @redundant_strided_load(ptr %src) { ; CHECK-NEXT: [[SRC_OFFSET:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 16 ; CHECK-NEXT: [[L:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC_OFFSET]], i32 8, i1 false, i32 4, i32 2) ; CHECK-NEXT: call void @llvm.matrix.column.major.store.v8f64.i32(<8 x double> [[L]], ptr [[SRC]], i32 8, i1 false, i32 4, i32 2) -; CHECK-NEXT: [[L_2:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC_OFFSET]], i32 8, i1 false, i32 4, i32 2) ; CHECK-NEXT: call void @use(<8 x double> [[L]]) -; CHECK-NEXT: call void @use(<8 x double> [[L_2]]) +; CHECK-NEXT: call void @use(<8 x double> [[L]]) ; CHECK-NEXT: ret void ; entry: @@ -75,9 +72,8 @@ define void @redundant_strided_load_non_matrix_store(ptr %src) { ; CHECK-NEXT: [[SRC_OFFSET:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 16 ; CHECK-NEXT: [[L:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC_OFFSET]], i32 8, i1 false, i32 4, i32 2) ; CHECK-NEXT: store double 4.200000e+01, ptr [[SRC]], align 8 -; CHECK-NEXT: [[L_2:%.*]] = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr [[SRC_OFFSET]], i32 8, i1 false, i32 4, i32 2) ; CHECK-NEXT: call void @use(<8 x double> [[L]]) -; CHECK-NEXT: call void @use(<8 x double> [[L_2]]) +; CHECK-NEXT: call void @use(<8 x double> [[L]]) ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll index ae8dc2d..005ca8c 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll @@ -175,18 +175,28 @@ define void @test_add_double_same_var_args_1(ptr %res, ptr noalias %A, ptr noali ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]] -; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = load <2 x double>, ptr [[TMP1]], align 4 -; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <2 x double>, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 4 +; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2> +; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3> +; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2> +; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3> +; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[STRIDED_VEC3]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[STRIDED_VEC1]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[STRIDED_VEC4]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]] -; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[TMP7]], align 4 -; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[TMP8]], align 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4 +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100 ; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: @@ -227,18 +237,28 @@ define void @test_add_double_same_var_args_2(ptr %res, ptr noalias %A, ptr noali ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]] -; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = load <2 x double>, ptr [[TMP1]], align 4 -; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <2 x double>, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 4 +; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2> +; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3> +; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2> +; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3> +; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC]] +; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC3]] ; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC1]] ; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC4]] ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]] -; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[TMP7]], align 4 -; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[TMP8]], align 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4 +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100 ; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll index b23702d..2a19402 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll @@ -319,46 +319,46 @@ define void @single_fmul_used_by_each_member(ptr noalias %A, ptr noalias %B, ptr ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 1 ; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], 6 ; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP33:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP21]] -; CHECK-NEXT: [[TMP37:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP20]] -; CHECK-NEXT: [[TMP39:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP22]] -; CHECK-NEXT: [[TMP24:%.*]] = load double, ptr [[TMP23]], align 8 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x double> poison, double [[TMP24]], i64 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT1]], <2 x double> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP25:%.*]] = load double, ptr [[TMP33]], align 8 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <2 x double> poison, double [[TMP25]], i64 0 -; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT12]], <2 x double> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP26:%.*]] = load double, ptr [[TMP37]], align 8 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <2 x double> poison, double [[TMP26]], i64 0 -; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT14]], <2 x double> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP27:%.*]] = load double, ptr [[TMP39]], align 8 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <2 x double> poison, double [[TMP27]], i64 0 -; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT16]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP25:%.*]] = getelementptr double, ptr [[TMP23]], i32 2 +; CHECK-NEXT: [[TMP26:%.*]] = getelementptr double, ptr [[TMP23]], i32 4 +; CHECK-NEXT: [[TMP27:%.*]] = getelementptr double, ptr [[TMP23]], i32 6 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP23]], align 8 +; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <2 x double>, ptr [[TMP25]], align 8 +; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <2 x double>, ptr [[TMP26]], align 8 +; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <2 x double>, ptr [[TMP27]], align 8 ; CHECK-NEXT: [[TMP28:%.*]] = fmul <2 x double> [[WIDE_LOAD]], splat (double 5.000000e+00) ; CHECK-NEXT: [[TMP29:%.*]] = fmul <2 x double> [[WIDE_LOAD12]], splat (double 5.000000e+00) ; CHECK-NEXT: [[TMP30:%.*]] = fmul <2 x double> [[WIDE_LOAD13]], splat (double 5.000000e+00) ; CHECK-NEXT: [[TMP31:%.*]] = fmul <2 x double> [[WIDE_LOAD14]], splat (double 5.000000e+00) ; CHECK-NEXT: [[TMP32:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP33:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP20]] ; CHECK-NEXT: [[TMP34:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP21]] -; CHECK-NEXT: [[TMP38:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP20]] ; CHECK-NEXT: [[TMP35:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP22]] -; CHECK-NEXT: store <2 x double> [[TMP28]], ptr [[TMP32]], align 8 -; CHECK-NEXT: store <2 x double> [[TMP29]], ptr [[TMP34]], align 8 -; CHECK-NEXT: store <2 x double> [[TMP30]], ptr [[TMP38]], align 8 -; CHECK-NEXT: store <2 x double> [[TMP31]], ptr [[TMP35]], align 8 +; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <2 x double> [[TMP28]], <2 x double> [[TMP28]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP36]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP32]], align 8 +; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <2 x double> [[TMP29]], <2 x double> [[TMP29]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; CHECK-NEXT: [[INTERLEAVED_VEC15:%.*]] = shufflevector <4 x double> [[TMP37]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC15]], ptr [[TMP33]], align 8 +; CHECK-NEXT: [[TMP38:%.*]] = shufflevector <2 x double> [[TMP30]], <2 x double> [[TMP30]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; CHECK-NEXT: [[INTERLEAVED_VEC16:%.*]] = shufflevector <4 x double> [[TMP38]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC16]], ptr [[TMP34]], align 8 +; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <2 x double> [[TMP31]], <2 x double> [[TMP31]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; CHECK-NEXT: [[INTERLEAVED_VEC17:%.*]] = shufflevector <4 x double> [[TMP39]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC17]], ptr [[TMP35]], align 8 ; CHECK-NEXT: [[TMP40:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP42:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP21]] ; CHECK-NEXT: [[TMP41:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP20]] +; CHECK-NEXT: [[TMP42:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP21]] ; CHECK-NEXT: [[TMP43:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP22]] -; CHECK-NEXT: store <2 x double> [[TMP28]], ptr [[TMP40]], align 8 -; CHECK-NEXT: store <2 x double> [[TMP29]], ptr [[TMP42]], align 8 -; CHECK-NEXT: store <2 x double> [[TMP30]], ptr [[TMP41]], align 8 -; CHECK-NEXT: store <2 x double> [[TMP31]], ptr [[TMP43]], align 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP40]], align 8 +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC15]], ptr [[TMP41]], align 8 +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC16]], ptr [[TMP42]], align 8 +; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC17]], ptr [[TMP43]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 ; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: @@ -435,7 +435,7 @@ exit: ret void } -; We should interleave by 2 after narrowing interleave groups to saturate +; FIXME: We should interleave by 2 after narrowing interleave groups to saturate ; load/store units. define void @test_interleave_after_narrowing(i32 %n, ptr %x, ptr noalias %y) { ; CHECK-LABEL: define void @test_interleave_after_narrowing( @@ -447,18 +447,12 @@ define void @test_interleave_after_narrowing(i32 %n, ptr %x, ptr noalias %y) { ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 4 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw float, ptr [[X]], i64 [[OFFSET_IDX]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[X]], i64 [[TMP5]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP7]], align 4 ; CHECK-NEXT: [[TMP1:%.*]] = fneg <4 x float> [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP4:%.*]] = fneg <4 x float> [[WIDE_LOAD1]] ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[Y]], i64 [[OFFSET_IDX]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw float, ptr [[Y]], i64 [[TMP5]] ; CHECK-NEXT: store <4 x float> [[TMP1]], ptr [[TMP2]], align 4 -; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[TMP6]], align 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1 ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 ; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll index 2865495..c261760 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll @@ -13,8 +13,12 @@ define void @load_store_interleave_group_tc_2(ptr noalias %data) { ; VF2: [[VECTOR_PH]]: ; VF2-NEXT: br label %[[VECTOR_BODY:.*]] ; VF2: [[VECTOR_BODY]]: -; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[DATA]], align 8 -; VF2-NEXT: store <2 x i64> [[WIDE_LOAD]], ptr [[DATA]], align 8 +; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[DATA]], align 8 +; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2> +; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3> +; VF2-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[DATA]], align 8 ; VF2-NEXT: br label %[[MIDDLE_BLOCK:.*]] ; VF2: [[MIDDLE_BLOCK]]: ; VF2-NEXT: br label %[[EXIT:.*]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll index 305a692..b63e03d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll @@ -1,81 +1,29 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5 -; RUN: opt -p loop-vectorize -force-vector-interleave=1 -S -mcpu=neoverse-512tvb %s | FileCheck --check-prefixes=IC1 %s -; RUN: opt -p loop-vectorize -S -mcpu=neoverse-512tvb %s | FileCheck --check-prefixes=CHECK %s +; RUN: opt -p loop-vectorize -force-vector-interleave=1 -S -mcpu=neoverse-512tvb %s | FileCheck --check-prefixes=CHECK %s target triple = "aarch64-unknown-linux" define void @load_store_interleave_group(ptr noalias %data) { -; IC1-LABEL: define void @load_store_interleave_group( -; IC1-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0:[0-9]+]] { -; IC1-NEXT: [[ENTRY:.*:]] -; IC1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() -; IC1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 -; IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]] -; IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] -; IC1: [[VECTOR_PH]]: -; IC1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() -; IC1-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2 -; IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]] -; IC1-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]] -; IC1-NEXT: br label %[[VECTOR_BODY:.*]] -; IC1: [[VECTOR_BODY]]: -; IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; IC1-NEXT: [[TMP4:%.*]] = shl nsw i64 [[INDEX]], 1 -; IC1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP4]] -; IC1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP5]], align 8 -; IC1-NEXT: store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 8 -; IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]] -; IC1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; IC1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; IC1: [[MIDDLE_BLOCK]]: -; IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]] -; IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] -; IC1: [[SCALAR_PH]]: -; ; CHECK-LABEL: define void @load_store_interleave_group( ; CHECK-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 3 +; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP5]] ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8 +; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]] -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[TMP2]], 0 -; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP20]], 1 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]] -; CHECK-NEXT: [[TMP24:%.*]] = mul i64 [[TMP2]], 2 -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP24]], 0 -; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1 -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP2]], 3 -; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], 0 -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 1 -; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP14]] ; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP17:%.*]] = shl nsw i64 [[TMP7]], 1 -; CHECK-NEXT: [[TMP18:%.*]] = shl nsw i64 [[TMP11]], 1 -; CHECK-NEXT: [[TMP19:%.*]] = shl nsw i64 [[TMP15]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]] -; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP17]] -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP18]] -; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP19]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP1]], align 8 -; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x i64>, ptr [[TMP21]], align 8 -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 2 x i64>, ptr [[TMP22]], align 8 -; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i64>, ptr [[TMP23]], align 8 ; CHECK-NEXT: store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8 -; CHECK-NEXT: store <vscale x 2 x i64> [[WIDE_LOAD1]], ptr [[TMP21]], align 8 -; CHECK-NEXT: store <vscale x 2 x i64> [[WIDE_LOAD2]], ptr [[TMP22]], align 8 -; CHECK-NEXT: store <vscale x 2 x i64> [[WIDE_LOAD3]], ptr [[TMP23]], align 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: @@ -105,82 +53,27 @@ exit: } define void @test_2xi64_unary_op_load_interleave_group(ptr noalias %data, ptr noalias %factor) { -; IC1-LABEL: define void @test_2xi64_unary_op_load_interleave_group( -; IC1-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]]) #[[ATTR0]] { -; IC1-NEXT: [[ENTRY:.*:]] -; IC1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() -; IC1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 -; IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1111, [[TMP1]] -; IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] -; IC1: [[VECTOR_PH]]: -; IC1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() -; IC1-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2 -; IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1111, [[TMP3]] -; IC1-NEXT: [[N_VEC:%.*]] = sub i64 1111, [[N_MOD_VF]] -; IC1-NEXT: br label %[[VECTOR_BODY:.*]] -; IC1: [[VECTOR_BODY]]: -; IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; IC1-NEXT: [[TMP4:%.*]] = shl nsw i64 [[INDEX]], 1 -; IC1-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP4]] -; IC1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP5]], align 8 -; IC1-NEXT: [[TMP6:%.*]] = fneg <vscale x 2 x double> [[WIDE_LOAD]] -; IC1-NEXT: store <vscale x 2 x double> [[TMP6]], ptr [[TMP5]], align 8 -; IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]] -; IC1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; IC1-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] -; IC1: [[MIDDLE_BLOCK]]: -; IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1111, [[N_VEC]] -; IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] -; IC1: [[SCALAR_PH]]: -; ; CHECK-LABEL: define void @test_2xi64_unary_op_load_interleave_group( ; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 3 +; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1111, [[TMP5]] ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8 +; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1111, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1111, [[N_MOD_VF]] -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[TMP2]], 0 -; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP20]], 1 -; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], [[TMP6]] -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP2]], 2 -; CHECK-NEXT: [[TMP28:%.*]] = add i64 [[TMP8]], 0 -; CHECK-NEXT: [[TMP29:%.*]] = mul i64 [[TMP28]], 1 -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP29]] -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP2]], 3 -; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], 0 -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 1 -; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP14]] ; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP17:%.*]] = shl nsw i64 [[TMP24]], 1 -; CHECK-NEXT: [[TMP18:%.*]] = shl nsw i64 [[TMP11]], 1 -; CHECK-NEXT: [[TMP19:%.*]] = shl nsw i64 [[TMP15]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]] -; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP17]] -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP18]] -; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP19]] ; CHECK-NEXT: [[TMP7:%.*]] = load <vscale x 2 x double>, ptr [[TMP1]], align 8 -; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP21]], align 8 -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 2 x double>, ptr [[TMP22]], align 8 -; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 2 x double>, ptr [[TMP23]], align 8 ; CHECK-NEXT: [[TMP9:%.*]] = fneg <vscale x 2 x double> [[TMP7]] -; CHECK-NEXT: [[TMP25:%.*]] = fneg <vscale x 2 x double> [[WIDE_LOAD1]] -; CHECK-NEXT: [[TMP26:%.*]] = fneg <vscale x 2 x double> [[WIDE_LOAD2]] -; CHECK-NEXT: [[TMP27:%.*]] = fneg <vscale x 2 x double> [[WIDE_LOAD3]] ; CHECK-NEXT: store <vscale x 2 x double> [[TMP9]], ptr [[TMP1]], align 8 -; CHECK-NEXT: store <vscale x 2 x double> [[TMP25]], ptr [[TMP21]], align 8 -; CHECK-NEXT: store <vscale x 2 x double> [[TMP26]], ptr [[TMP22]], align 8 -; CHECK-NEXT: store <vscale x 2 x double> [[TMP27]], ptr [[TMP23]], align 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll index abfb44d..d290f2d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll @@ -60,26 +60,32 @@ define void @test_2xi64_with_wide_load(ptr noalias %data, ptr noalias %factor) { ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[TMP0]] -; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP2]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP3:%.*]] = load i64, ptr [[TMP4]], align 8 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT2]], <2 x i64> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 2 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8 +; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8 ; CHECK-NEXT: [[TMP6:%.*]] = shl nsw i64 [[INDEX]], 1 ; CHECK-NEXT: [[TMP7:%.*]] = shl nsw i64 [[TMP0]], 1 ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP6]] ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP7]] -; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = load <2 x i64>, ptr [[TMP8]], align 8 -; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = load <2 x i64>, ptr [[TMP9]], align 8 +; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2> +; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3> +; CHECK-NEXT: [[WIDE_VEC3:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8 +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC3]], <4 x i64> poison, <2 x i32> <i32 0, i32 2> +; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <4 x i64> [[WIDE_VEC3]], <4 x i64> poison, <2 x i32> <i32 1, i32 3> +; CHECK-NEXT: [[TMP10:%.*]] = mul <2 x i64> [[BROADCAST_SPLAT]], [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP11:%.*]] = mul <2 x i64> [[BROADCAST_SPLAT3]], [[WIDE_LOAD1]] ; CHECK-NEXT: [[TMP15:%.*]] = mul <2 x i64> [[BROADCAST_SPLAT]], [[STRIDED_VEC2]] ; CHECK-NEXT: [[TMP16:%.*]] = mul <2 x i64> [[BROADCAST_SPLAT3]], [[STRIDED_VEC5]] -; CHECK-NEXT: store <2 x i64> [[TMP15]], ptr [[TMP8]], align 8 -; CHECK-NEXT: store <2 x i64> [[TMP16]], ptr [[TMP9]], align 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x i64> [[TMP10]], <2 x i64> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP17]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; CHECK-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <2 x i64> [[TMP11]], <2 x i64> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; CHECK-NEXT: [[INTERLEAVED_VEC6:%.*]] = shufflevector <4 x i64> [[TMP18]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; CHECK-NEXT: store <4 x i64> [[INTERLEAVED_VEC6]], ptr [[TMP9]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100 ; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll index f2e689c..75980ba 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll @@ -328,8 +328,10 @@ define void @same_live_in_store_interleave_group(i64 %x, ptr noalias %dst) { ; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; VF2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1 ; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]] -; VF2-NEXT: store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 8 -; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1 +; VF2-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> +; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3> +; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8 +; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; VF2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100 ; VF2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; VF2: [[MIDDLE_BLOCK]]: diff --git a/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll b/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll index e42e2c7..b26e9cf 100644 --- a/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll +++ b/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll @@ -1779,7 +1779,7 @@ for.body: ; preds = %entry, %for.body ; CHECK: LV: Scalar loop costs: 24 ; CHECK: LV: Vector loop of width 2 costs: 33 ; CHECK: LV: Vector loop of width 4 costs: 30 -; CHECK: LV: Selecting VF: 4 +; CHECK: LV: Selecting VF: 1 define hidden void @four_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) { entry: %cmp45.not = icmp eq i32 %N, 0 diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll index f4d80af..2a3ce03 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll @@ -8,69 +8,15 @@ target triple = "x86_64-unknown-linux" define void @test_4xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n) { ; CHECK-LABEL: define void @test_4xi64( ; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[ITER_CHECK:.*]]: +; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] -; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: -; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[N_MOD_VF1:%.*]] = urem i64 [[N]], 16 -; CHECK-NEXT: [[N_VEC1:%.*]] = sub i64 [[N]], [[N_MOD_VF1]] -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3 -; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[TMP0]] -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[TMP1]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[TMP2]] -; CHECK-NEXT: [[TMP7:%.*]] = load i64, ptr [[TMP20]], align 8 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[TMP7]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP21]], align 8 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT5]], <4 x i64> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP22]], align 8 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT7]], <4 x i64> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT9]], <4 x i64> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[INDEX]], i32 0 -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[TMP1]], i32 0 -; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[TMP2]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP11]], align 8 -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP12]], align 8 -; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i64>, ptr [[TMP13]], align 8 -; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i64>, ptr [[TMP23]], align 8 -; CHECK-NEXT: [[TMP15:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT1]], [[WIDE_LOAD1]] -; CHECK-NEXT: [[TMP16:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT6]], [[WIDE_LOAD2]] -; CHECK-NEXT: [[TMP17:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT8]], [[WIDE_LOAD3]] -; CHECK-NEXT: [[TMP18:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT10]], [[WIDE_LOAD4]] -; CHECK-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP11]], align 8 -; CHECK-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP12]], align 8 -; CHECK-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP13]], align 8 -; CHECK-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP23]], align 8 -; CHECK-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT1]], [[N_VEC1]] -; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N]], [[N_VEC1]] -; CHECK-NEXT: br i1 [[CMP_N1]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] -; CHECK: [[VEC_EPILOG_ITER_CHECK]]: -; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF1]], 4 -; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]] -; CHECK: [[VEC_EPILOG_PH]]: -; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] -; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] -; CHECK: [[VEC_EPILOG_VECTOR_BODY]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]] ; CHECK-NEXT: [[TMP5:%.*]] = load i64, ptr [[ARRAYIDX]], align 8 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0 @@ -81,15 +27,15 @@ define void @test_4xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n) ; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP3]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 1 ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP14]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] -; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: +; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]] -; CHECK: [[VEC_EPILOG_SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] +; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] ; CHECK-NEXT: [[DATA_2:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV1]] ; CHECK-NEXT: [[L_2:%.*]] = load i64, ptr [[DATA_2]], align 8 ; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[IV1]], i32 0 @@ -110,7 +56,7 @@ define void @test_4xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n) ; CHECK-NEXT: store i64 [[MUL_3]], ptr [[DATA_3]], align 8 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -171,7 +117,7 @@ define void @test_2xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n) ; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] @@ -194,7 +140,7 @@ define void @test_2xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n) ; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -249,7 +195,7 @@ define void @test_2xi64_interleave_loads_order_flipped(ptr noalias %data, ptr no ; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] @@ -272,7 +218,7 @@ define void @test_2xi64_interleave_loads_order_flipped(ptr noalias %data, ptr no ; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -327,7 +273,7 @@ define void @test_2xi64_store_order_flipped_1(ptr noalias %data, ptr noalias %fa ; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] @@ -350,7 +296,7 @@ define void @test_2xi64_store_order_flipped_1(ptr noalias %data, ptr noalias %fa ; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_1]], align 8 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -405,7 +351,7 @@ define void @test_2xi64_store_order_flipped_2(ptr noalias %data, ptr noalias %fa ; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] @@ -428,7 +374,7 @@ define void @test_2xi64_store_order_flipped_2(ptr noalias %data, ptr noalias %fa ; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_0]], align 8 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -489,7 +435,7 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal ; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP16]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: br label %[[SCALAR_PH]] ; CHECK: [[SCALAR_PH]]: @@ -513,7 +459,7 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal ; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -573,7 +519,7 @@ define void @test_3xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n) ; CHECK-NEXT: store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] @@ -598,7 +544,7 @@ define void @test_3xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n) ; CHECK-NEXT: store i64 [[MUL_2]], ptr [[DATA_2]], align 8 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP16:![0-9]+]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -707,7 +653,7 @@ define void @test_3xi32(ptr noalias %data, ptr noalias %factor, i64 noundef %n) ; CHECK-NEXT: store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8 ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: br label %[[SCALAR_PH]] ; CHECK: [[SCALAR_PH]]: @@ -731,7 +677,7 @@ define void @test_3xi32(ptr noalias %data, ptr noalias %factor, i64 noundef %n) ; CHECK-NEXT: store i32 [[MUL_2]], ptr [[DATA_2]], align 8 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP18:![0-9]+]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -765,20 +711,19 @@ exit: ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} -; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]} -; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]} -; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]} -; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]} -; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]} -; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]} -; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]} -; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]} -; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]} -; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META2]], [[META1]]} -; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]], [[META2]]} -; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META2]], [[META1]]} -; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]], [[META2]]} -; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META2]], [[META1]]} -; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]]} -; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META2]], [[META1]]} +; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]} +; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} +; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]} +; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]} +; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]} +; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]} +; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]} +; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]} +; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]} +; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]} +; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]} +; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]} +; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]} +; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]} +; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]} ;. diff --git a/llvm/test/Transforms/Util/dbg-user-of-aext.ll b/llvm/test/Transforms/Util/dbg-user-of-aext.ll index 9e7935e..b3d1b90 100644 --- a/llvm/test/Transforms/Util/dbg-user-of-aext.ll +++ b/llvm/test/Transforms/Util/dbg-user-of-aext.ll @@ -27,7 +27,7 @@ %struct.foo = type { i8, i64 } ; Function Attrs: noinline nounwind uwtable -define void @_Z1fbb3foo(i1 zeroext %b, i1 zeroext %frag, i8 %g.coerce0, i64 %g.coerce1) #0 !dbg !6 { +define void @_Z1fbb3foo(i1 zeroext %b, i1 zeroext %frag, i8 %g.coerce0, i64 %g.coerce1) !dbg !6 { entry: %g = alloca %struct.foo, align 8 %b.addr = alloca i8, align 1 @@ -51,10 +51,7 @@ entry: ; CHECK: ![[VAR_FRAG]] = !DILocalVariable(name: "frag" ; Function Attrs: nounwind readnone speculatable -declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 - -attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind readnone speculatable } +declare void @llvm.dbg.declare(metadata, metadata, metadata) !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} diff --git a/llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll b/llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll index ad23bf7..e9f0c8c 100644 --- a/llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll +++ b/llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll @@ -19,18 +19,18 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-unknown" ; Function Attrs: nounwind -define float @fn(float %f) #0 { +define float @fn(float %f) { ; CHECK: define float @fn( ; CHECK: call fast float @expf( %f.addr = alloca float, align 4 store float %f, ptr %f.addr, align 4, !tbaa !1 %1 = load float, ptr %f.addr, align 4, !tbaa !1 - %call = call fast float @expf(float %1) #3 + %call = call fast float @expf(float %1) ret float %call } ; Function Attrs: inlinehint nounwind readnone -define available_externally float @expf(float %x) #1 { +define available_externally float @expf(float %x) { ; CHECK: define available_externally float @expf( ; CHECK: fpext float ; CHECK: call fast double @exp( @@ -39,17 +39,13 @@ define available_externally float @expf(float %x) #1 { store float %x, ptr %x.addr, align 4, !tbaa !1 %1 = load float, ptr %x.addr, align 4, !tbaa !1 %conv = fpext float %1 to double - %call = call fast double @exp(double %conv) #3 + %call = call fast double @exp(double %conv) %conv1 = fptrunc double %call to float ret float %conv1 } ; Function Attrs: nounwind readnone -declare double @exp(double) #2 - -attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { inlinehint nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #2 = { nounwind readnone } +declare double @exp(double) !llvm.ident = !{!0} diff --git a/llvm/test/tools/llvm-ir2vec/embeddings-flowaware.ll b/llvm/test/tools/llvm-ir2vec/embeddings-flowaware.ll index b2362f8..ade228d 100644 --- a/llvm/test/tools/llvm-ir2vec/embeddings-flowaware.ll +++ b/llvm/test/tools/llvm-ir2vec/embeddings-flowaware.ll @@ -49,7 +49,7 @@ entry: ; CHECK-FUNC-LEVEL-ABC: Function: abc ; CHECK-FUNC-LEVEL-NEXT-ABC: [ 3630.00 3672.00 3714.00 ] -; CHECK-FUNC-DEF: Error: Function 'def' not found +; CHECK-FUNC-DEF: error: Function 'def' not found ; CHECK-BB-LEVEL: Function: abc ; CHECK-BB-LEVEL-NEXT: entry: [ 3630.00 3672.00 3714.00 ] diff --git a/llvm/test/tools/llvm-ir2vec/embeddings-symbolic.ll b/llvm/test/tools/llvm-ir2vec/embeddings-symbolic.ll index f9aa108..9d60e12 100644 --- a/llvm/test/tools/llvm-ir2vec/embeddings-symbolic.ll +++ b/llvm/test/tools/llvm-ir2vec/embeddings-symbolic.ll @@ -49,7 +49,7 @@ entry: ; CHECK-FUNC-LEVEL-ABC: Function: abc ; CHECK-FUNC-LEVEL-NEXT-ABC: [ 878.00 889.00 900.00 ] -; CHECK-FUNC-DEF: Error: Function 'def' not found +; CHECK-FUNC-DEF: error: Function 'def' not found ; CHECK-BB-LEVEL: Function: abc ; CHECK-BB-LEVEL-NEXT: entry: [ 878.00 889.00 900.00 ] diff --git a/llvm/test/tools/llvm-ir2vec/embeddings-symbolic.mir b/llvm/test/tools/llvm-ir2vec/embeddings-symbolic.mir new file mode 100644 index 0000000..ef835fe --- /dev/null +++ b/llvm/test/tools/llvm-ir2vec/embeddings-symbolic.mir @@ -0,0 +1,92 @@ +# REQUIRES: x86_64-linux +# RUN: llvm-ir2vec embeddings --mode=mir --mir2vec-vocab-path=%S/../../CodeGen/MIR2Vec/Inputs/mir2vec_dummy_3D_vocab.json %s | FileCheck %s -check-prefix=CHECK-DEFAULT +# RUN: llvm-ir2vec embeddings --mode=mir --level=func --mir2vec-vocab-path=%S/../../CodeGen/MIR2Vec/Inputs/mir2vec_dummy_3D_vocab.json %s | FileCheck %s -check-prefix=CHECK-FUNC-LEVEL +# RUN: llvm-ir2vec embeddings --mode=mir --level=func --function=add_function --mir2vec-vocab-path=%S/../../CodeGen/MIR2Vec/Inputs/mir2vec_dummy_3D_vocab.json %s | FileCheck %s -check-prefix=CHECK-FUNC-LEVEL-ADD +# RUN: not llvm-ir2vec embeddings --mode=mir --level=func --function=missing_function --mir2vec-vocab-path=%S/../../CodeGen/MIR2Vec/Inputs/mir2vec_dummy_3D_vocab.json %s 2>&1 | FileCheck %s -check-prefix=CHECK-FUNC-MISSING +# RUN: llvm-ir2vec embeddings --mode=mir --level=bb --mir2vec-vocab-path=%S/../../CodeGen/MIR2Vec/Inputs/mir2vec_dummy_3D_vocab.json %s | FileCheck %s -check-prefix=CHECK-BB-LEVEL +# RUN: llvm-ir2vec embeddings --mode=mir --level=inst --function=add_function --mir2vec-vocab-path=%S/../../CodeGen/MIR2Vec/Inputs/mir2vec_dummy_3D_vocab.json %s | FileCheck %s -check-prefix=CHECK-INST-LEVEL + +--- | + target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" + target triple = "x86_64-unknown-linux-gnu" + + define dso_local noundef i32 @add_function(i32 noundef %a, i32 noundef %b) { + entry: + %sum = add nsw i32 %a, %b + %result = mul nsw i32 %sum, 2 + ret i32 %result + } + + define dso_local void @simple_function() { + entry: + ret void + } +... +--- +name: add_function +alignment: 16 +tracksRegLiveness: true +registers: + - { id: 0, class: gr32 } + - { id: 1, class: gr32 } + - { id: 2, class: gr32 } + - { id: 3, class: gr32 } +liveins: + - { reg: '$edi', virtual-reg: '%0' } + - { reg: '$esi', virtual-reg: '%1' } +body: | + bb.0.entry: + liveins: $edi, $esi + + %1:gr32 = COPY $esi + %0:gr32 = COPY $edi + %2:gr32 = nsw ADD32rr %0, %1, implicit-def dead $eflags + %3:gr32 = ADD32rr %2, %2, implicit-def dead $eflags + $eax = COPY %3 + RET 0, $eax + +--- +name: simple_function +alignment: 16 +tracksRegLiveness: true +body: | + bb.0.entry: + RET 0 + +# CHECK-DEFAULT: MIR2Vec embeddings for machine function add_function: +# CHECK-DEFAULT-NEXT: Function vector: [ 26.50 27.10 27.70 ] +# CHECK-DEFAULT: MIR2Vec embeddings for machine function simple_function: +# CHECK-DEFAULT-NEXT: Function vector: [ 1.10 1.20 1.30 ] + +# CHECK-FUNC-LEVEL: MIR2Vec embeddings for machine function add_function: +# CHECK-FUNC-LEVEL-NEXT: Function vector: [ 26.50 27.10 27.70 ] +# CHECK-FUNC-LEVEL: MIR2Vec embeddings for machine function simple_function: +# CHECK-FUNC-LEVEL-NEXT: Function vector: [ 1.10 1.20 1.30 ] + +# CHECK-FUNC-LEVEL-ADD: MIR2Vec embeddings for machine function add_function: +# CHECK-FUNC-LEVEL-ADD-NEXT: Function vector: [ 26.50 27.10 27.70 ] +# CHECK-FUNC-LEVEL-ADD-NOT: simple_function + +# CHECK-FUNC-MISSING: error: Function 'missing_function' not found + +# CHECK-BB-LEVEL: MIR2Vec embeddings for machine function add_function: +# CHECK-BB-LEVEL-NEXT: Basic block vectors: +# CHECK-BB-LEVEL-NEXT: MBB entry: [ 26.50 27.10 27.70 ] +# CHECK-BB-LEVEL: MIR2Vec embeddings for machine function simple_function: +# CHECK-BB-LEVEL-NEXT: Basic block vectors: +# CHECK-BB-LEVEL-NEXT: MBB entry: [ 1.10 1.20 1.30 ] + +# CHECK-INST-LEVEL: MIR2Vec embeddings for machine function add_function: +# CHECK-INST-LEVEL-NEXT: Instruction vectors: +# CHECK-INST-LEVEL: %1:gr32 = COPY $esi +# CHECK-INST-LEVEL-NEXT: -> [ 6.00 6.10 6.20 ] +# CHECK-INST-LEVEL-NEXT: %0:gr32 = COPY $edi +# CHECK-INST-LEVEL-NEXT: -> [ 6.00 6.10 6.20 ] +# CHECK-INST-LEVEL: %2:gr32 = nsw ADD32rr +# CHECK-INST-LEVEL: -> [ 3.70 3.80 3.90 ] +# CHECK-INST-LEVEL: %3:gr32 = ADD32rr +# CHECK-INST-LEVEL: -> [ 3.70 3.80 3.90 ] +# CHECK-INST-LEVEL: $eax = COPY %3:gr32 +# CHECK-INST-LEVEL-NEXT: -> [ 6.00 6.10 6.20 ] +# CHECK-INST-LEVEL: RET 0, $eax +# CHECK-INST-LEVEL-NEXT: -> [ 1.10 1.20 1.30 ] diff --git a/llvm/test/tools/llvm-ir2vec/error-handling.ll b/llvm/test/tools/llvm-ir2vec/error-handling.ll index b944ea0..8e9e455 100644 --- a/llvm/test/tools/llvm-ir2vec/error-handling.ll +++ b/llvm/test/tools/llvm-ir2vec/error-handling.ll @@ -10,4 +10,4 @@ entry: } ; CHECK-NO-VOCAB: error: IR2Vec vocabulary file path not specified; You may need to set it using --ir2vec-vocab-path -; CHECK-FUNC-NOT-FOUND: Error: Function 'nonexistent' not found +; CHECK-FUNC-NOT-FOUND: error: Function 'nonexistent' not found diff --git a/llvm/test/tools/llvm-ir2vec/error-handling.mir b/llvm/test/tools/llvm-ir2vec/error-handling.mir new file mode 100644 index 0000000..caec454c --- /dev/null +++ b/llvm/test/tools/llvm-ir2vec/error-handling.mir @@ -0,0 +1,41 @@ +# REQUIRES: x86_64-linux +# Test error handling and input validation for llvm-ir2vec tool in MIR mode + +# RUN: not llvm-ir2vec embeddings --mode=mir %s 2>&1 | FileCheck %s -check-prefix=CHECK-NO-VOCAB +# RUN: not llvm-ir2vec embeddings --mode=mir --mir2vec-vocab-path=%S/nonexistent-vocab.json %s 2>&1 | FileCheck %s -check-prefix=CHECK-VOCAB-NOT-FOUND +# RUN: not llvm-ir2vec embeddings --mode=mir --mir2vec-vocab-path=%S/../../CodeGen/MIR2Vec/Inputs/mir2vec_invalid_vocab.json %s 2>&1 | FileCheck %s -check-prefix=CHECK-INVALID-VOCAB +# RUN: not llvm-ir2vec embeddings --mode=mir --function=nonexistent_function --mir2vec-vocab-path=%S/../../CodeGen/MIR2Vec/Inputs/mir2vec_dummy_3D_vocab.json %s 2>&1 | FileCheck %s -check-prefix=CHECK-FUNC-NOT-FOUND + +--- | + target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" + target triple = "x86_64-unknown-linux-gnu" + + define dso_local noundef i32 @test_function(i32 noundef %a) { + entry: + ret i32 %a + } +... +--- +name: test_function +alignment: 16 +tracksRegLiveness: true +registers: + - { id: 0, class: gr32 } +liveins: + - { reg: '$edi', virtual-reg: '%0' } +body: | + bb.0.entry: + liveins: $edi + + %0:gr32 = COPY $edi + $eax = COPY %0 + RET 0, $eax + +# CHECK-NO-VOCAB: error: Failed to load MIR2Vec vocabulary - MIR2Vec vocabulary file path not specified; set it using --mir2vec-vocab-path + +# CHECK-VOCAB-NOT-FOUND: error: Failed to load MIR2Vec vocabulary +# CHECK-VOCAB-NOT-FOUND: No such file or directory + +# CHECK-INVALID-VOCAB: error: Failed to load MIR2Vec vocabulary - Missing 'Opcodes' section in vocabulary file + +# CHECK-FUNC-NOT-FOUND: error: Function 'nonexistent_function' not found diff --git a/llvm/test/tools/llvm-objdump/MachO/disassemble-source-dsym.test b/llvm/test/tools/llvm-objdump/MachO/disassemble-source-dsym.test index aaaf6bf..9899dc5 100644 --- a/llvm/test/tools/llvm-objdump/MachO/disassemble-source-dsym.test +++ b/llvm/test/tools/llvm-objdump/MachO/disassemble-source-dsym.test @@ -13,4 +13,35 @@ # RUN: dsymutil -f -oso-prepend-path=%p/../../dsymutil/ %t3 -o %t3.dSYM # RUN: llvm-objdump --source --prefix=%p/../../dsymutil %t3 | FileCheck --check-prefix=SOURCE %s +## Test that --source works with --macho flag. + +## --macho w/ explicit .dSYM +# RUN: llvm-objdump < %p/../../dsymutil/Inputs/basic.macho.x86_64 - --source --macho --dsym=%t1.dSYM --prefix=%p/../../dsymutil | \ +# RUN: FileCheck --check-prefix=SOURCE %s + +## --macho w/ auto-detected .dSYM (dir) +# RUN: llvm-objdump --source --macho --prefix=%p/../../dsymutil %t2 | FileCheck --check-prefix=SOURCE %s + +## --macho w/ auto-detected .dSYM (file) +# RUN: llvm-objdump --source --macho --prefix=%p/../../dsymutil %t3 | FileCheck --check-prefix=SOURCE %s + # SOURCE: ; int bar(int arg) { + +## Test that --line-numbers works with --macho flag. + +## --macho -l w/ explicit .dSYM +# RUN: llvm-objdump -d -l --macho --dsym=%t1.dSYM %p/../../dsymutil/Inputs/basic.macho.x86_64 | FileCheck --check-prefix=LINE %s + +## --macho -l w/ object file (embedded debug info) +# RUN: llvm-objdump -d -l --macho %p/../../dsymutil/Inputs/basic1.macho.x86_64.o | FileCheck --check-prefix=LINE_OBJ %s + +# LINE: (__TEXT,__text) section +# LINE: _bar: +# LINE: ; bar(): +# LINE: ; {{.*}}basic3.c: + +# LINE_OBJ: (__TEXT,__text) section +# LINE_OBJ: _main: +# LINE_OBJ: ; main(): +# LINE_OBJ: ; {{.*}}basic1.c:23 +# LINE_OBJ: pushq %rbp ## basic1.c:23:0 diff --git a/llvm/test/tools/llvm-readobj/ELF/section-types.test b/llvm/test/tools/llvm-readobj/ELF/section-types.test index 904892a..12a9d05 100644 --- a/llvm/test/tools/llvm-readobj/ELF/section-types.test +++ b/llvm/test/tools/llvm-readobj/ELF/section-types.test @@ -63,6 +63,8 @@ # LLVM: Type: SHT_LLVM_PART_PHDR # LLVM: Name: .llvm.lto # LLVM: Type: SHT_LLVM_LTO +# LLVM: Name: .llvm.callgraph +# LLVM: Type: SHT_LLVM_CALL_GRAPH # LLVM: Name: gnu_sframe # LLVM: Type: SHT_GNU_SFRAME # LLVM: Name: gnu_attributes @@ -127,6 +129,7 @@ # GNU-NEXT: part1 LLVM_PART_EHDR # GNU-NEXT: .phdrs LLVM_PART_PHDR # GNU-NEXT: .llvm.lto LLVM_LTO +# GNU-NEXT: .llvm.callgraph LLVM_CALL_GRAPH # GNU-NEXT: gnu_sframe SFRAME # GNU-NEXT: gnu_attributes ATTRIBUTES # GNU-NEXT: gnu_hash GNU_HASH @@ -218,6 +221,8 @@ Sections: Type: SHT_LLVM_PART_PHDR - Name: .llvm.lto Type: SHT_LLVM_LTO + - Name: .llvm.callgraph + Type: SHT_LLVM_CALL_GRAPH - Name: gnu_sframe Type: SHT_GNU_SFRAME - Name: gnu_attributes diff --git a/llvm/tools/llvm-ir2vec/CMakeLists.txt b/llvm/tools/llvm-ir2vec/CMakeLists.txt index a4cf969..2bb6686 100644 --- a/llvm/tools/llvm-ir2vec/CMakeLists.txt +++ b/llvm/tools/llvm-ir2vec/CMakeLists.txt @@ -1,10 +1,25 @@ set(LLVM_LINK_COMPONENTS + # Core LLVM components for IR processing Analysis Core IRReader Support + + # Machine IR components (for -mode=mir) + CodeGen + MIRParser + + # Target initialization (required for MIR parsing) + AllTargetsAsmParsers + AllTargetsCodeGens + AllTargetsDescs + AllTargetsInfos + TargetParser ) add_llvm_tool(llvm-ir2vec llvm-ir2vec.cpp + + DEPENDS + intrinsics_gen ) diff --git a/llvm/tools/llvm-ir2vec/llvm-ir2vec.cpp b/llvm/tools/llvm-ir2vec/llvm-ir2vec.cpp index 1031932..a723d37 100644 --- a/llvm/tools/llvm-ir2vec/llvm-ir2vec.cpp +++ b/llvm/tools/llvm-ir2vec/llvm-ir2vec.cpp @@ -1,4 +1,4 @@ -//===- llvm-ir2vec.cpp - IR2Vec Embedding Generation Tool -----------------===// +//===- llvm-ir2vec.cpp - IR2Vec/MIR2Vec Embedding Generation Tool --------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -7,9 +7,13 @@ //===----------------------------------------------------------------------===// /// /// \file -/// This file implements the IR2Vec embedding generation tool. +/// This file implements the IR2Vec and MIR2Vec embedding generation tool. /// -/// This tool provides three main subcommands: +/// This tool supports two modes: +/// - LLVM IR mode (-mode=llvm): Process LLVM IR +/// - Machine IR mode (-mode=mir): Process Machine IR +/// +/// Available subcommands: /// /// 1. Triplet Generation (triplets): /// Generates numeric triplets (head, tail, relation) for vocabulary @@ -23,16 +27,24 @@ /// Usage: llvm-ir2vec entities input.bc -o entity2id.txt /// /// 3. Embedding Generation (embeddings): -/// Generates IR2Vec embeddings using a trained vocabulary. -/// Usage: llvm-ir2vec embeddings --ir2vec-vocab-path=vocab.json -/// --ir2vec-kind=<kind> --level=<level> input.bc -o embeddings.txt -/// Kind: --ir2vec-kind=symbolic (default), --ir2vec-kind=flow-aware +/// Generates IR2Vec/MIR2Vec embeddings using a trained vocabulary. +/// +/// For LLVM IR: +/// llvm-ir2vec embeddings --ir2vec-vocab-path=vocab.json +/// --ir2vec-kind=<kind> --level=<level> input.bc -o embeddings.txt +/// Kind: --ir2vec-kind=symbolic (default), --ir2vec-kind=flow-aware +/// +/// For Machine IR: +/// llvm-ir2vec embeddings -mode=mir --mir2vec-vocab-path=vocab.json +/// --level=<level> input.mir -o embeddings.txt +/// /// Levels: --level=inst (instructions), --level=bb (basic blocks), -/// --level=func (functions) (See IR2Vec.cpp for more embedding generation -/// options) +/// --level=func (functions) (See IR2Vec.cpp/MIR2Vec.cpp for more embedding +/// generation options) /// //===----------------------------------------------------------------------===// +#include "llvm/ADT/ArrayRef.h" #include "llvm/Analysis/IR2Vec.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Function.h" @@ -50,10 +62,38 @@ #include "llvm/Support/SourceMgr.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/CodeGen/MIR2Vec.h" +#include "llvm/CodeGen/MIRParser/MIRParser.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/WithColor.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/TargetParser/Host.h" + #define DEBUG_TYPE "ir2vec" namespace llvm { -namespace ir2vec { + +static const char *ToolName = "llvm-ir2vec"; + +// Common option category for options shared between IR2Vec and MIR2Vec +static cl::OptionCategory CommonCategory("Common Options", + "Options applicable to both IR2Vec " + "and MIR2Vec modes"); + +enum IRKind { + LLVMIR = 0, ///< LLVM IR + MIR ///< Machine IR +}; + +static cl::opt<IRKind> + IRMode("mode", cl::desc("Tool operation mode:"), + cl::values(clEnumValN(LLVMIR, "llvm", "Process LLVM IR"), + clEnumValN(MIR, "mir", "Process Machine IR")), + cl::init(LLVMIR), cl::cat(CommonCategory)); // Subcommands static cl::SubCommand @@ -70,18 +110,18 @@ static cl::opt<std::string> InputFilename(cl::Positional, cl::desc("<input bitcode file or '-' for stdin>"), cl::init("-"), cl::sub(TripletsSubCmd), - cl::sub(EmbeddingsSubCmd), cl::cat(ir2vec::IR2VecCategory)); + cl::sub(EmbeddingsSubCmd), cl::cat(CommonCategory)); static cl::opt<std::string> OutputFilename("o", cl::desc("Output filename"), cl::value_desc("filename"), cl::init("-"), - cl::cat(ir2vec::IR2VecCategory)); + cl::cat(CommonCategory)); // Embedding-specific options static cl::opt<std::string> FunctionName("function", cl::desc("Process specific function only"), cl::value_desc("name"), cl::Optional, cl::init(""), - cl::sub(EmbeddingsSubCmd), cl::cat(ir2vec::IR2VecCategory)); + cl::sub(EmbeddingsSubCmd), cl::cat(CommonCategory)); enum EmbeddingLevel { InstructionLevel, // Generate instruction-level embeddings @@ -98,9 +138,9 @@ static cl::opt<EmbeddingLevel> clEnumValN(FunctionLevel, "func", "Generate function-level embeddings")), cl::init(FunctionLevel), cl::sub(EmbeddingsSubCmd), - cl::cat(ir2vec::IR2VecCategory)); + cl::cat(CommonCategory)); -namespace { +namespace ir2vec { /// Relation types for triplet generation enum RelationType { @@ -220,7 +260,8 @@ public: /// Generate embeddings for the entire module void generateEmbeddings(raw_ostream &OS) const { if (!Vocab->isValid()) { - OS << "Error: Vocabulary is not valid. IR2VecTool not initialized.\n"; + WithColor::error(errs(), ToolName) + << "Vocabulary is not valid. IR2VecTool not initialized.\n"; return; } @@ -239,8 +280,8 @@ public: assert(Vocab->isValid() && "Vocabulary is not valid"); auto Emb = Embedder::create(IR2VecEmbeddingKind, F, *Vocab); if (!Emb) { - OS << "Error: Failed to create embedder for function " << F.getName() - << "\n"; + WithColor::error(errs(), ToolName) + << "Failed to create embedder for function " << F.getName() << "\n"; return; } @@ -300,20 +341,119 @@ Error processModule(Module &M, raw_ostream &OS) { } return Error::success(); } -} // namespace } // namespace ir2vec + +namespace mir2vec { + +/// Helper class for MIR2Vec embedding generation +class MIR2VecTool { +private: + MachineModuleInfo &MMI; + std::unique_ptr<MIRVocabulary> Vocab; + +public: + explicit MIR2VecTool(MachineModuleInfo &MMI) : MMI(MMI) {} + + /// Initialize MIR2Vec vocabulary + bool initializeVocabulary(const Module &M) { + MIR2VecVocabProvider Provider(MMI); + auto VocabOrErr = Provider.getVocabulary(M); + if (!VocabOrErr) { + WithColor::error(errs(), ToolName) + << "Failed to load MIR2Vec vocabulary - " + << toString(VocabOrErr.takeError()) << "\n"; + return false; + } + Vocab = std::make_unique<MIRVocabulary>(std::move(*VocabOrErr)); + return true; + } + + /// Generate embeddings for all machine functions in the module + void generateEmbeddings(const Module &M, raw_ostream &OS) const { + if (!Vocab) { + WithColor::error(errs(), ToolName) << "Vocabulary not initialized.\n"; + return; + } + + for (const Function &F : M) { + if (F.isDeclaration()) + continue; + + MachineFunction *MF = MMI.getMachineFunction(F); + if (!MF) { + WithColor::warning(errs(), ToolName) + << "No MachineFunction for " << F.getName() << "\n"; + continue; + } + + generateEmbeddings(*MF, OS); + } + } + + /// Generate embeddings for a specific machine function + void generateEmbeddings(MachineFunction &MF, raw_ostream &OS) const { + if (!Vocab) { + WithColor::error(errs(), ToolName) << "Vocabulary not initialized.\n"; + return; + } + + auto Emb = MIREmbedder::create(MIR2VecKind::Symbolic, MF, *Vocab); + if (!Emb) { + WithColor::error(errs(), ToolName) + << "Failed to create embedder for " << MF.getName() << "\n"; + return; + } + + OS << "MIR2Vec embeddings for machine function " << MF.getName() << ":\n"; + + // Generate embeddings based on the specified level + switch (Level) { + case FunctionLevel: { + OS << "Function vector: "; + Emb->getMFunctionVector().print(OS); + break; + } + case BasicBlockLevel: { + OS << "Basic block vectors:\n"; + for (const MachineBasicBlock &MBB : MF) { + OS << "MBB " << MBB.getName() << ": "; + Emb->getMBBVector(MBB).print(OS); + } + break; + } + case InstructionLevel: { + OS << "Instruction vectors:\n"; + for (const MachineBasicBlock &MBB : MF) { + for (const MachineInstr &MI : MBB) { + OS << MI << " -> "; + Emb->getMInstVector(MI).print(OS); + } + } + break; + } + } + } + + const MIRVocabulary *getVocabulary() const { return Vocab.get(); } +}; + +} // namespace mir2vec + } // namespace llvm int main(int argc, char **argv) { using namespace llvm; using namespace llvm::ir2vec; + using namespace llvm::mir2vec; InitLLVM X(argc, argv); - cl::HideUnrelatedOptions(ir2vec::IR2VecCategory); + // Show Common, IR2Vec and MIR2Vec option categories + cl::HideUnrelatedOptions(ArrayRef<const cl::OptionCategory *>{ + &CommonCategory, &ir2vec::IR2VecCategory, &mir2vec::MIR2VecCategory}); cl::ParseCommandLineOptions( argc, argv, - "IR2Vec - Embedding Generation Tool\n" - "Generates embeddings for a given LLVM IR and " + "IR2Vec/MIR2Vec - Embedding Generation Tool\n" + "Generates embeddings for a given LLVM IR or MIR and " "supports triplet generation for vocabulary " "training and embedding generation.\n\n" "See https://llvm.org/docs/CommandGuide/llvm-ir2vec.html for more " @@ -322,30 +462,117 @@ int main(int argc, char **argv) { std::error_code EC; raw_fd_ostream OS(OutputFilename, EC); if (EC) { - errs() << "Error opening output file: " << EC.message() << "\n"; + WithColor::error(errs(), ToolName) + << "opening output file: " << EC.message() << "\n"; return 1; } - if (EntitiesSubCmd) { - // Just dump entity mappings without processing any IR - IR2VecTool::generateEntityMappings(OS); + if (IRMode == IRKind::LLVMIR) { + if (EntitiesSubCmd) { + // Just dump entity mappings without processing any IR + IR2VecTool::generateEntityMappings(OS); + return 0; + } + + // Parse the input LLVM IR file or stdin + SMDiagnostic Err; + LLVMContext Context; + std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context); + if (!M) { + Err.print(ToolName, errs()); + return 1; + } + + if (Error Err = processModule(*M, OS)) { + handleAllErrors(std::move(Err), [&](const ErrorInfoBase &EIB) { + WithColor::error(errs(), ToolName) << EIB.message() << "\n"; + }); + return 1; + } return 0; } + if (IRMode == IRKind::MIR) { + // Initialize targets for Machine IR processing + InitializeAllTargets(); + InitializeAllTargetMCs(); + InitializeAllAsmParsers(); + InitializeAllAsmPrinters(); + static codegen::RegisterCodeGenFlags CGF; + + // Parse MIR input file + SMDiagnostic Err; + LLVMContext Context; + std::unique_ptr<TargetMachine> TM; + + auto MIR = createMIRParserFromFile(InputFilename, Err, Context); + if (!MIR) { + Err.print(ToolName, errs()); + return 1; + } - // Parse the input LLVM IR file or stdin - SMDiagnostic Err; - LLVMContext Context; - std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context); - if (!M) { - Err.print(argv[0], errs()); - return 1; - } + auto SetDataLayout = [&](StringRef DataLayoutTargetTriple, + StringRef OldDLStr) -> std::optional<std::string> { + std::string IRTargetTriple = DataLayoutTargetTriple.str(); + Triple TheTriple = Triple(IRTargetTriple); + if (TheTriple.getTriple().empty()) + TheTriple.setTriple(sys::getDefaultTargetTriple()); + auto TMOrErr = codegen::createTargetMachineForTriple(TheTriple.str()); + if (!TMOrErr) { + Err.print(ToolName, errs()); + exit(1); + } + TM = std::move(*TMOrErr); + return TM->createDataLayout().getStringRepresentation(); + }; + + std::unique_ptr<Module> M = MIR->parseIRModule(SetDataLayout); + if (!M) { + Err.print(ToolName, errs()); + return 1; + } - if (Error Err = processModule(*M, OS)) { - handleAllErrors(std::move(Err), [&](const ErrorInfoBase &EIB) { - errs() << "Error: " << EIB.message() << "\n"; - }); - return 1; + // Parse machine functions + auto MMI = std::make_unique<MachineModuleInfo>(TM.get()); + if (!MMI || MIR->parseMachineFunctions(*M, *MMI)) { + Err.print(ToolName, errs()); + return 1; + } + + // Create MIR2Vec tool and initialize vocabulary + MIR2VecTool Tool(*MMI); + if (!Tool.initializeVocabulary(*M)) + return 1; + + LLVM_DEBUG(dbgs() << "MIR2Vec vocabulary loaded successfully.\n" + << "Vocabulary dimension: " + << Tool.getVocabulary()->getDimension() << "\n" + << "Vocabulary size: " + << Tool.getVocabulary()->getCanonicalSize() << "\n"); + + // Generate embeddings based on subcommand + if (!FunctionName.empty()) { + // Process single function + Function *F = M->getFunction(FunctionName); + if (!F) { + WithColor::error(errs(), ToolName) + << "Function '" << FunctionName << "' not found\n"; + return 1; + } + + MachineFunction *MF = MMI->getMachineFunction(*F); + if (!MF) { + WithColor::error(errs(), ToolName) + << "No MachineFunction for " << FunctionName << "\n"; + return 1; + } + + Tool.generateEmbeddings(*MF, OS); + } else { + // Process all functions + Tool.generateEmbeddings(*M, OS); + } + + return 0; } return 0; diff --git a/llvm/tools/llvm-objdump/MachODump.cpp b/llvm/tools/llvm-objdump/MachODump.cpp index 8e9c91f..f633ed5 100644 --- a/llvm/tools/llvm-objdump/MachODump.cpp +++ b/llvm/tools/llvm-objdump/MachODump.cpp @@ -13,6 +13,7 @@ #include "MachODump.h" #include "ObjdumpOptID.h" +#include "SourcePrinter.h" #include "llvm-objdump.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" @@ -7415,18 +7416,28 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF, std::unique_ptr<DIContext> diContext; std::unique_ptr<Binary> DSYMBinary; std::unique_ptr<MemoryBuffer> DSYMBuf; - if (UseDbg) { - // If separate DSym file path was specified, parse it as a macho file, - // get the sections and supply it to the section name parsing machinery. - if (const ObjectFile *DbgObj = - getMachODSymObject(MachOOF, Filename, DSYMBinary, DSYMBuf)) { + const ObjectFile *DbgObj = MachOOF; + if (UseDbg || PrintSource || PrintLines) { + // Look for debug info in external dSYM file or embedded in the object. + // getMachODSymObject returns MachOOF by default if no external dSYM found. + const ObjectFile *DSym = + getMachODSymObject(MachOOF, Filename, DSYMBinary, DSYMBuf); + if (!DSym) + return; + DbgObj = DSym; + if (UseDbg || PrintLines) { // Setup the DIContext diContext = DWARFContext::create(*DbgObj); - } else { - return; } } + std::optional<SourcePrinter> SP; + std::optional<LiveElementPrinter> LEP; + if (PrintSource || PrintLines) { + SP.emplace(DbgObj, TheTarget->getName()); + LEP.emplace(*MRI, *STI); + } + if (FilterSections.empty()) outs() << "(" << DisSegName << "," << DisSectName << ") section\n"; @@ -7605,6 +7616,12 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF, outs() << SymName << ":\n"; uint64_t PC = SectAddress + Index; + + if (PrintSource || PrintLines) { + formatted_raw_ostream FOS(outs()); + SP->printSourceLine(FOS, {PC, SectIdx}, Filename, *LEP); + } + if (LeadingAddr) { if (FullLeadingAddr) { if (MachOOF->is64Bit()) @@ -7696,6 +7713,11 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF, uint64_t PC = SectAddress + Index; + if (PrintSource || PrintLines) { + formatted_raw_ostream FOS(outs()); + SP->printSourceLine(FOS, {PC, SectIdx}, Filename, *LEP); + } + if (DumpAndSkipDataInCode(PC, Bytes.data() + Index, Dices, InstSize)) continue; diff --git a/llvm/unittests/ADT/SmallVectorTest.cpp b/llvm/unittests/ADT/SmallVectorTest.cpp index e2e778f..b216359 100644 --- a/llvm/unittests/ADT/SmallVectorTest.cpp +++ b/llvm/unittests/ADT/SmallVectorTest.cpp @@ -599,6 +599,15 @@ TYPED_TEST(SmallVectorTest, AssignSmallVector) { assertValuesInOrder(V, 2u, 7, 7); } +TYPED_TEST(SmallVectorTest, AssignArrayRef) { + SCOPED_TRACE("AssignArrayRef"); + auto &V = this->theVector; + Constructable Other[] = {7, 8, 9}; + V.push_back(Constructable(1)); + V.assign(ArrayRef(Other)); + assertValuesInOrder(V, 3u, 7, 8, 9); +} + // Move-assign test TYPED_TEST(SmallVectorTest, MoveAssignTest) { SCOPED_TRACE("MoveAssignTest"); diff --git a/llvm/unittests/CAS/CASTestConfig.cpp b/llvm/unittests/CAS/CASTestConfig.cpp index 91d0970..10e4b68 100644 --- a/llvm/unittests/CAS/CASTestConfig.cpp +++ b/llvm/unittests/CAS/CASTestConfig.cpp @@ -9,6 +9,7 @@ #include "CASTestConfig.h" #include "llvm/CAS/ObjectStore.h" #include "gtest/gtest.h" +#include <mutex> using namespace llvm; using namespace llvm::cas; diff --git a/llvm/unittests/Support/GlobPatternTest.cpp b/llvm/unittests/Support/GlobPatternTest.cpp index 58fd767..872a21e 100644 --- a/llvm/unittests/Support/GlobPatternTest.cpp +++ b/llvm/unittests/Support/GlobPatternTest.cpp @@ -329,6 +329,72 @@ TEST_F(GlobPatternTest, PrefixSuffix) { EXPECT_EQ("cd", Pat->suffix()); } +TEST_F(GlobPatternTest, Substr) { + auto Pat = GlobPattern::create(""); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("", Pat->longest_substr()); + + Pat = GlobPattern::create("abcd"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bcd"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("", Pat->longest_substr()); + + Pat = GlobPattern::create("*abcd"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("", Pat->longest_substr()); + + Pat = GlobPattern::create("abcd*"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bc*d"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("bc", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bc*def*g"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("def", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bcd*ef*g"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("bcd", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bcd*efg*h"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("bcd", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bcd[ef]g*h"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("bcd", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bc[d]efg*h"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("efg", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bc[]]efg*h"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("efg", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bcde\\fg*h"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("bcde", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bcde\\[fg*h"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("bcde", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bcde?fg*h"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("bcde", Pat->longest_substr()); + + Pat = GlobPattern::create("a*bcdef{g}*h"); + ASSERT_TRUE((bool)Pat); + EXPECT_EQ("bcdef", Pat->longest_substr()); +} + TEST_F(GlobPatternTest, Pathological) { std::string P, S(40, 'a'); StringRef Pieces[] = {"a*", "[ba]*", "{b*,a*}*"}; diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn index 022cd87..1ca9292 100644 --- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn @@ -301,7 +301,6 @@ write_cmake_config("llvm-config") { "LLVM_BUILD_SHARED_LIBS=", "LLVM_ENABLE_LLVM_C_EXPORT_ANNOTATIONS=", "LLVM_ENABLE_TELEMETRY=", - "LLVM_ENABLE_ONDISK_CAS=", "LLVM_DEFAULT_TARGET_TRIPLE=$llvm_target_triple", "LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE=", "LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN=", @@ -367,6 +366,12 @@ write_cmake_config("llvm-config") { values += [ "LLVM_ENABLE_DIA_SDK=" ] } + if (llvm_enable_ondisk_cas) { + values += [ "LLVM_ENABLE_ONDISK_CAS=1" ] + } else { + values += [ "LLVM_ENABLE_ONDISK_CAS=" ] + } + if (llvm_enable_threads) { values += [ "LLVM_ENABLE_THREADS=1" ] } else { diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/config.gni b/llvm/utils/gn/secondary/llvm/include/llvm/Config/config.gni index 8c2ab8a..715b03e 100644 --- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/config.gni +++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/config.gni @@ -1,4 +1,7 @@ declare_args() { # Iterate unordered llvm containers in reverse. llvm_enable_reverse_iteration = false + + # Iterate unordered llvm containers in reverse. + llvm_enable_ondisk_cas = false } diff --git a/llvm/utils/gn/secondary/llvm/lib/AsmParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/AsmParser/BUILD.gn index 9ff794f..2ad867d 100644 --- a/llvm/utils/gn/secondary/llvm/lib/AsmParser/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/AsmParser/BUILD.gn @@ -6,6 +6,7 @@ static_library("AsmParser") { "//llvm/lib/Support", ] sources = [ + "AsmParserContext.cpp", "LLLexer.cpp", "LLParser.cpp", "Parser.cpp", diff --git a/llvm/utils/gn/secondary/llvm/lib/CAS/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CAS/BUILD.gn index b4edd8d..5590b27 100644 --- a/llvm/utils/gn/secondary/llvm/lib/CAS/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/CAS/BUILD.gn @@ -10,6 +10,8 @@ static_library("CAS") { "ObjectStore.cpp", "OnDiskCommon.cpp", "OnDiskDataAllocator.cpp", + "OnDiskGraphDB.cpp", + "OnDiskKeyValueDB.cpp", "OnDiskTrieRawHashMap.cpp", ] } diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn index c89e335..e47ca1e 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn @@ -137,6 +137,7 @@ static_library("LLVMAMDGPUCodeGen") { "AMDGPUAsmPrinter.cpp", "AMDGPUAtomicOptimizer.cpp", "AMDGPUAttributor.cpp", + "AMDGPUBarrierLatency.cpp", "AMDGPUCallLowering.cpp", "AMDGPUCodeGenPrepare.cpp", "AMDGPUCombinerHelper.cpp", diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/Hexagon/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/Hexagon/BUILD.gn index c055001..065d33d 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/Hexagon/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/Hexagon/BUILD.gn @@ -79,6 +79,7 @@ static_library("LLVMHexagonCodeGen") { "HexagonOptAddrMode.cpp", "HexagonOptimizeSZextends.cpp", "HexagonPeephole.cpp", + "HexagonQFPOptimizer.cpp", "HexagonRDFOpt.cpp", "HexagonRegisterInfo.cpp", "HexagonSelectionDAGInfo.cpp", diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn index 07a7951..4d75201 100644 --- a/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn @@ -1,9 +1,16 @@ executable("llvm-ir2vec") { deps = [ "//llvm/lib/Analysis", + "//llvm/lib/CodeGen", + "//llvm/lib/CodeGen/MIRParser", "//llvm/lib/IR", "//llvm/lib/IRReader", "//llvm/lib/Support", + "//llvm/lib/Target:AllTargetsAsmParsers", + "//llvm/lib/Target:AllTargetsCodeGens", + "//llvm/lib/Target:AllTargetsDescs", + "//llvm/lib/Target:AllTargetsInfos", + "//llvm/lib/TargetParser", ] sources = [ "llvm-ir2vec.cpp" ] } diff --git a/llvm/utils/gn/secondary/llvm/unittests/CAS/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/CAS/BUILD.gn index 52a64be..2d9eb68 100644 --- a/llvm/utils/gn/secondary/llvm/unittests/CAS/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/unittests/CAS/BUILD.gn @@ -1,3 +1,4 @@ +import("//llvm/include/llvm/Config/config.gni") import("//third-party/unittest/unittest.gni") unittest("CASTests") { @@ -10,8 +11,15 @@ unittest("CASTests") { "ActionCacheTest.cpp", "CASTestConfig.cpp", "ObjectStoreTest.cpp", - "OnDiskDataAllocatorTest.cpp", - "OnDiskTrieRawHashMapTest.cpp", - "ProgramTest.cpp", ] + + if (llvm_enable_ondisk_cas) { + sources += [ + "OnDiskDataAllocatorTest.cpp", + "OnDiskGraphDBTest.cpp", + "OnDiskKeyValueDBTest.cpp", + "OnDiskTrieRawHashMapTest.cpp", + "ProgramTest.cpp", + ] + } } diff --git a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn index 376f689..dfe6d6d 100644 --- a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn @@ -42,6 +42,7 @@ unittest("OrcJITTests") { "SymbolStringPoolTest.cpp", "TaskDispatchTest.cpp", "ThreadSafeModuleTest.cpp", + "WaitingOnGraphTest.cpp", "WrapperFunctionUtilsTest.cpp", ] diff --git a/llvm/utils/lit/tests/shtest-ulimit.py b/llvm/utils/lit/tests/shtest-ulimit.py index 9a8febd..dadde70 100644 --- a/llvm/utils/lit/tests/shtest-ulimit.py +++ b/llvm/utils/lit/tests/shtest-ulimit.py @@ -5,7 +5,11 @@ # as well. # UNSUPPORTED: system-windows, system-solaris -# RUN: not %{lit} -a -v %{inputs}/shtest-ulimit --order=lexical | FileCheck %s +# RUN: %{python} %S/Inputs/shtest-ulimit/print_limits.py | grep RLIMIT_NOFILE \ +# RUN: | sed -n -e 's/.*=//p' | tr -d '\n' > %t.nofile_limit + +# RUN: not %{lit} -a -v %{inputs}/shtest-ulimit --order=lexical \ +# RUN: | FileCheck -DBASE_NOFILE_LIMIT=%{readfile:%t.nofile_limit} %s # CHECK: -- Testing: 3 tests{{.*}} @@ -18,4 +22,4 @@ # CHECK: RLIMIT_NOFILE=50 # CHECK-LABEL: FAIL: shtest-ulimit :: ulimit_reset.txt ({{[^)]*}}) -# CHECK-NOT: RLIMIT_NOFILE=50 +# CHECK: RLIMIT_NOFILE=[[BASE_NOFILE_LIMIT]] diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h index e2a60f5..05d2316 100644 --- a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h @@ -46,10 +46,10 @@ mlir::acc::CopyinOp, mlir::acc::CreateOp, mlir::acc::PresentOp, \ mlir::acc::NoCreateOp, mlir::acc::AttachOp, mlir::acc::DevicePtrOp, \ mlir::acc::GetDevicePtrOp, mlir::acc::PrivateOp, \ - mlir::acc::FirstprivateOp, mlir::acc::UpdateDeviceOp, \ - mlir::acc::UseDeviceOp, mlir::acc::ReductionOp, \ - mlir::acc::DeclareDeviceResidentOp, mlir::acc::DeclareLinkOp, \ - mlir::acc::CacheOp + mlir::acc::FirstprivateOp, mlir::acc::FirstprivateMapInitialOp, \ + mlir::acc::UpdateDeviceOp, mlir::acc::UseDeviceOp, \ + mlir::acc::ReductionOp, mlir::acc::DeclareDeviceResidentOp, \ + mlir::acc::DeclareLinkOp, mlir::acc::CacheOp #define ACC_DATA_EXIT_OPS \ mlir::acc::CopyoutOp, mlir::acc::DeleteOp, mlir::acc::DetachOp, \ mlir::acc::UpdateHostOp diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td index e78cdbe..2f87975 100644 --- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td @@ -787,6 +787,21 @@ def OpenACC_FirstprivateOp : OpenACC_DataEntryOp<"firstprivate", let extraClassDeclaration = extraClassDeclarationBase; } +// The mapping of firstprivate cannot be represented through an `acc.copyin` +// since that operation includes present counter updates (and private variables +// do not impact counters). Instead, the below operation is used to represent +// the mapping of that initial value which can be used to initialize the private +// copies. +def OpenACC_FirstprivateMapInitialOp : OpenACC_DataEntryOp<"firstprivate_map", + "mlir::acc::DataClause::acc_firstprivate", "", [], + (ins Arg<OpenACC_AnyPointerOrMappableType,"Host variable",[MemRead]>:$var)> { + let summary = "Used to decompose firstprivate semantics and represents the " + "mapping of the initial value."; + let results = (outs Arg<OpenACC_AnyPointerOrMappableType, + "Accelerator mapped variable",[MemWrite]>:$accVar); + let extraClassDeclaration = extraClassDeclarationBase; +} + //===----------------------------------------------------------------------===// // 2.5.15 reduction clause //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/IR/Builders.h b/mlir/include/mlir/IR/Builders.h index 9205f16..3ba6818 100644 --- a/mlir/include/mlir/IR/Builders.h +++ b/mlir/include/mlir/IR/Builders.h @@ -502,6 +502,7 @@ private: public: /// Create an operation of specific op type at the current insertion point. template <typename OpTy, typename... Args> + [[deprecated("Use OpTy::create instead")]] OpTy create(Location location, Args &&...args) { OperationState state(location, getCheckRegisteredInfo<OpTy>(location.getContext())); @@ -517,9 +518,9 @@ public: /// the results of the operation. /// /// Note: This performs opportunistic eager folding during IR construction. - /// The folders are designed to operate efficiently on canonical IR, which + /// The folders are designed to operate efficiently on canonical IR, which /// this API does not enforce. Complete folding isn't only expected in the - /// context of canonicalization which intertwine folders with pattern + /// context of canonicalization which intertwine folders with pattern /// rewrites until fixed-point. template <typename OpTy, typename... Args> void createOrFold(SmallVectorImpl<Value> &results, Location location, diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp index 94947b7..c06a48e 100644 --- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp +++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp @@ -1437,6 +1437,13 @@ ExtractStridedMetadataOp::fold(FoldAdaptor adaptor, atLeastOneReplacement |= replaceConstantUsesOf( builder, getLoc(), getStrides(), getConstifiedMixedStrides()); + // extract_strided_metadata(cast(x)) -> extract_strided_metadata(x). + if (auto prev = getSource().getDefiningOp<CastOp>()) + if (isa<MemRefType>(prev.getSource().getType())) { + getSourceMutable().assign(prev.getSource()); + atLeastOneReplacement = true; + } + return success(atLeastOneReplacement); } diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp index d35566a..bd02516 100644 --- a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp +++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp @@ -1033,91 +1033,6 @@ class ExtractStridedMetadataOpReinterpretCastFolder } }; -/// Replace `base, offset, sizes, strides = -/// extract_strided_metadata( -/// cast(src) to dstTy)` -/// With -/// ``` -/// base, ... = extract_strided_metadata(src) -/// offset = !dstTy.srcOffset.isDynamic() -/// ? dstTy.srcOffset -/// : extract_strided_metadata(src).offset -/// sizes = for each srcSize in dstTy.srcSizes: -/// !srcSize.isDynamic() -/// ? srcSize -// : extract_strided_metadata(src).sizes[i] -/// strides = for each srcStride in dstTy.srcStrides: -/// !srcStrides.isDynamic() -/// ? srcStrides -/// : extract_strided_metadata(src).strides[i] -/// ``` -/// -/// In other words, consume the `cast` and apply its effects -/// on the offset, sizes, and strides or compute them directly from `src`. -class ExtractStridedMetadataOpCastFolder - : public OpRewritePattern<memref::ExtractStridedMetadataOp> { - using OpRewritePattern::OpRewritePattern; - - LogicalResult - matchAndRewrite(memref::ExtractStridedMetadataOp extractStridedMetadataOp, - PatternRewriter &rewriter) const override { - Value source = extractStridedMetadataOp.getSource(); - auto castOp = source.getDefiningOp<memref::CastOp>(); - if (!castOp) - return failure(); - - Location loc = extractStridedMetadataOp.getLoc(); - // Check if the source is suitable for extract_strided_metadata. - SmallVector<Type> inferredReturnTypes; - if (failed(extractStridedMetadataOp.inferReturnTypes( - rewriter.getContext(), loc, {castOp.getSource()}, - /*attributes=*/{}, /*properties=*/nullptr, /*regions=*/{}, - inferredReturnTypes))) - return rewriter.notifyMatchFailure(castOp, - "cast source's type is incompatible"); - - auto memrefType = cast<MemRefType>(source.getType()); - unsigned rank = memrefType.getRank(); - SmallVector<OpFoldResult> results; - results.resize_for_overwrite(rank * 2 + 2); - - auto newExtractStridedMetadata = memref::ExtractStridedMetadataOp::create( - rewriter, loc, castOp.getSource()); - - // Register the base_buffer. - results[0] = newExtractStridedMetadata.getBaseBuffer(); - - auto getConstantOrValue = [&rewriter](int64_t constant, - OpFoldResult ofr) -> OpFoldResult { - return ShapedType::isStatic(constant) - ? OpFoldResult(rewriter.getIndexAttr(constant)) - : ofr; - }; - - auto [sourceStrides, sourceOffset] = memrefType.getStridesAndOffset(); - assert(sourceStrides.size() == rank && "unexpected number of strides"); - - // Register the new offset. - results[1] = - getConstantOrValue(sourceOffset, newExtractStridedMetadata.getOffset()); - - const unsigned sizeStartIdx = 2; - const unsigned strideStartIdx = sizeStartIdx + rank; - ArrayRef<int64_t> sourceSizes = memrefType.getShape(); - - SmallVector<OpFoldResult> sizes = newExtractStridedMetadata.getSizes(); - SmallVector<OpFoldResult> strides = newExtractStridedMetadata.getStrides(); - for (unsigned i = 0; i < rank; ++i) { - results[sizeStartIdx + i] = getConstantOrValue(sourceSizes[i], sizes[i]); - results[strideStartIdx + i] = - getConstantOrValue(sourceStrides[i], strides[i]); - } - rewriter.replaceOp(extractStridedMetadataOp, - getValueOrCreateConstantIndexOp(rewriter, loc, results)); - return success(); - } -}; - /// Replace `base, offset, sizes, strides = extract_strided_metadata( /// memory_space_cast(src) to dstTy)` /// with @@ -1209,7 +1124,6 @@ void memref::populateExpandStridedMetadataPatterns( RewriteExtractAlignedPointerAsIndexOfViewLikeOp, ExtractStridedMetadataOpReinterpretCastFolder, ExtractStridedMetadataOpSubviewFolder, - ExtractStridedMetadataOpCastFolder, ExtractStridedMetadataOpMemorySpaceCastFolder, ExtractStridedMetadataOpAssumeAlignmentFolder, ExtractStridedMetadataOpExtractStridedMetadataFolder>( @@ -1226,7 +1140,6 @@ void memref::populateResolveExtractStridedMetadataPatterns( ExtractStridedMetadataOpSubviewFolder, RewriteExtractAlignedPointerAsIndexOfViewLikeOp, ExtractStridedMetadataOpReinterpretCastFolder, - ExtractStridedMetadataOpCastFolder, ExtractStridedMetadataOpMemorySpaceCastFolder, ExtractStridedMetadataOpAssumeAlignmentFolder, ExtractStridedMetadataOpExtractStridedMetadataFolder>( diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp index 5ca0100..ca46629 100644 --- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp +++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp @@ -610,6 +610,20 @@ LogicalResult acc::FirstprivateOp::verify() { } //===----------------------------------------------------------------------===// +// FirstprivateMapInitialOp +//===----------------------------------------------------------------------===// +LogicalResult acc::FirstprivateMapInitialOp::verify() { + if (getDataClause() != acc::DataClause::acc_firstprivate) + return emitError("data clause associated with firstprivate operation must " + "match its intent"); + if (failed(checkVarAndVarType(*this))) + return failure(); + if (failed(checkNoModifier(*this))) + return failure(); + return success(); +} + +//===----------------------------------------------------------------------===// // ReductionOp //===----------------------------------------------------------------------===// LogicalResult acc::ReductionOp::verify() { diff --git a/mlir/lib/Dialect/OpenMP/Transforms/OpenMPOffloadPrivatizationPrepare.cpp b/mlir/lib/Dialect/OpenMP/Transforms/OpenMPOffloadPrivatizationPrepare.cpp index db54eaa..a9125ec 100644 --- a/mlir/lib/Dialect/OpenMP/Transforms/OpenMPOffloadPrivatizationPrepare.cpp +++ b/mlir/lib/Dialect/OpenMP/Transforms/OpenMPOffloadPrivatizationPrepare.cpp @@ -205,14 +205,14 @@ class PrepareForOMPOffloadPrivatizationPass assert(!region.empty() && "region cannot be empty"); LLVM::LLVMFuncOp func = createFuncOpForRegion( loc, mod, region, funcName, rewriter, returnsValue); - auto call = rewriter.create<LLVM::CallOp>(loc, func, args); + auto call = LLVM::CallOp::create(rewriter, loc, func, args); return call.getResult(); }; Value moldArg, newArg; if (isPrivatizedByValue) { - moldArg = rewriter.create<LLVM::LoadOp>(loc, varType, varPtr); - newArg = rewriter.create<LLVM::LoadOp>(loc, varType, heapMem); + moldArg = LLVM::LoadOp::create(rewriter, loc, varType, varPtr); + newArg = LLVM::LoadOp::create(rewriter, loc, varType, heapMem); } else { moldArg = varPtr; newArg = heapMem; @@ -234,7 +234,7 @@ class PrepareForOMPOffloadPrivatizationPass {moldArg, initializedVal}, /*returnsValue=*/true); if (isPrivatizedByValue) - (void)rewriter.create<LLVM::StoreOp>(loc, initializedVal, heapMem); + (void)LLVM::StoreOp::create(rewriter, loc, initializedVal, heapMem); // clone origOp, replace all uses of varPtr with heapMem and // erase origOp. @@ -252,14 +252,17 @@ class PrepareForOMPOffloadPrivatizationPass // variable, rewrite all the uses of the original variable with // the heap-allocated variable. rewriter.setInsertionPoint(targetOp); - rewriter.setInsertionPoint(cloneModifyAndErase(mapInfoOp)); + mapInfoOp = cast<omp::MapInfoOp>(cloneModifyAndErase(mapInfoOp)); + rewriter.setInsertionPoint(mapInfoOp); // Fix any members that may use varPtr to now use heapMem for (auto member : mapInfoOp.getMembers()) { auto memberMapInfoOp = cast<omp::MapInfoOp>(member.getDefiningOp()); if (!usesVarPtr(memberMapInfoOp)) continue; - rewriter.setInsertionPoint(cloneModifyAndErase(memberMapInfoOp)); + memberMapInfoOp = + cast<omp::MapInfoOp>(cloneModifyAndErase(memberMapInfoOp)); + rewriter.setInsertionPoint(memberMapInfoOp); if (memberMapInfoOp.getVarPtrPtr()) { Operation *varPtrPtrdefOp = @@ -275,8 +278,8 @@ class PrepareForOMPOffloadPrivatizationPass // targetOp. if (isPrivatizedByValue) { rewriter.setInsertionPoint(targetOp); - auto newPrivVar = rewriter.create<LLVM::LoadOp>(mapInfoOp.getLoc(), - varType, heapMem); + auto newPrivVar = LLVM::LoadOp::create(rewriter, mapInfoOp.getLoc(), + varType, heapMem); newPrivVars.push_back(newPrivVar); } @@ -294,7 +297,7 @@ class PrepareForOMPOffloadPrivatizationPass cleanupTaskOp = omp::TaskOp::create(rewriter, loc, taskOperands); Block *taskBlock = rewriter.createBlock(&cleanupTaskOp.getRegion()); rewriter.setInsertionPointToEnd(taskBlock); - rewriter.create<omp::TerminatorOp>(cleanupTaskOp.getLoc()); + omp::TerminatorOp::create(rewriter, cleanupTaskOp.getLoc()); } rewriter.setInsertionPointToStart( &*cleanupTaskOp.getRegion().getBlocks().begin()); @@ -307,8 +310,8 @@ class PrepareForOMPOffloadPrivatizationPass LLVM::lookupOrCreateFreeFn(rewriter, mod); assert(llvm::succeeded(freeFunc) && "Could not find free in the module"); - (void)rewriter.create<LLVM::CallOp>(loc, freeFunc.value(), - ValueRange{heapMem}); + (void)LLVM::CallOp::create(rewriter, loc, freeFunc.value(), + ValueRange{heapMem}); } } assert(newPrivVars.size() == privateVars.size() && @@ -390,11 +393,11 @@ private: const DataLayout &dl = DataLayout(mod); std::int64_t distance = getSizeInBytes(dl, varType); - Value sizeBytes = rewriter.create<LLVM::ConstantOp>( - loc, mallocFn.getFunctionType().getParamType(0), distance); + Value sizeBytes = LLVM::ConstantOp::create( + rewriter, loc, mallocFn.getFunctionType().getParamType(0), distance); auto mallocCallOp = - rewriter.create<LLVM::CallOp>(loc, mallocFn, ValueRange{sizeBytes}); + LLVM::CallOp::create(rewriter, loc, mallocFn, ValueRange{sizeBytes}); return mallocCallOp.getResult(); } diff --git a/mlir/test/Dialect/MemRef/canonicalize.mlir b/mlir/test/Dialect/MemRef/canonicalize.mlir index 7160b52..3130902 100644 --- a/mlir/test/Dialect/MemRef/canonicalize.mlir +++ b/mlir/test/Dialect/MemRef/canonicalize.mlir @@ -901,6 +901,132 @@ func.func @scope_merge_without_terminator() { // ----- +// Check that we simplify extract_strided_metadata of cast +// when the source of the cast is compatible with what +// `extract_strided_metadata`s accept. +// +// When we apply the transformation the resulting offset, sizes and strides +// should come straight from the inputs of the cast. +// Additionally the folder on extract_strided_metadata should propagate the +// static information. +// +// CHECK-LABEL: func @extract_strided_metadata_of_cast +// CHECK-SAME: %[[ARG:.*]]: memref<3x?xi32, strided<[4, ?], offset: ?>>) +// +// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index +// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : index +// CHECK: %[[BASE:.*]], %[[DYN_OFFSET:.*]], %[[DYN_SIZES:.*]]:2, %[[DYN_STRIDES:.*]]:2 = memref.extract_strided_metadata %[[ARG]] +// +// CHECK: return %[[BASE]], %[[DYN_OFFSET]], %[[C3]], %[[DYN_SIZES]]#1, %[[C4]], %[[DYN_STRIDES]]#1 +func.func @extract_strided_metadata_of_cast( + %arg : memref<3x?xi32, strided<[4, ?], offset:?>>) + -> (memref<i32>, index, + index, index, + index, index) { + + %cast = + memref.cast %arg : + memref<3x?xi32, strided<[4, ?], offset: ?>> to + memref<?x?xi32, strided<[?, ?], offset: ?>> + + %base, %base_offset, %sizes:2, %strides:2 = + memref.extract_strided_metadata %cast:memref<?x?xi32, strided<[?, ?], offset: ?>> + -> memref<i32>, index, + index, index, + index, index + + return %base, %base_offset, + %sizes#0, %sizes#1, + %strides#0, %strides#1 : + memref<i32>, index, + index, index, + index, index +} + +// ----- + +// Check that we simplify extract_strided_metadata of cast +// when the source of the cast is compatible with what +// `extract_strided_metadata`s accept. +// +// Same as extract_strided_metadata_of_cast but with constant sizes and strides +// in the destination type. +// +// CHECK-LABEL: func @extract_strided_metadata_of_cast_w_csts +// CHECK-SAME: %[[ARG:.*]]: memref<?x?xi32, strided<[?, ?], offset: ?>>) +// +// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : index +// CHECK-DAG: %[[C18:.*]] = arith.constant 18 : index +// CHECK-DAG: %[[C25:.*]] = arith.constant 25 : index +// CHECK: %[[BASE:.*]], %[[DYN_OFFSET:.*]], %[[DYN_SIZES:.*]]:2, %[[DYN_STRIDES:.*]]:2 = memref.extract_strided_metadata %[[ARG]] +// +// CHECK: return %[[BASE]], %[[C25]], %[[C4]], %[[DYN_SIZES]]#1, %[[DYN_STRIDES]]#0, %[[C18]] +func.func @extract_strided_metadata_of_cast_w_csts( + %arg : memref<?x?xi32, strided<[?, ?], offset:?>>) + -> (memref<i32>, index, + index, index, + index, index) { + + %cast = + memref.cast %arg : + memref<?x?xi32, strided<[?, ?], offset: ?>> to + memref<4x?xi32, strided<[?, 18], offset: 25>> + + %base, %base_offset, %sizes:2, %strides:2 = + memref.extract_strided_metadata %cast:memref<4x?xi32, strided<[?, 18], offset: 25>> + -> memref<i32>, index, + index, index, + index, index + + return %base, %base_offset, + %sizes#0, %sizes#1, + %strides#0, %strides#1 : + memref<i32>, index, + index, index, + index, index +} + +// ----- + +// Check that we don't simplify extract_strided_metadata of +// cast when the source of the cast is unranked. +// Unranked memrefs cannot feed into extract_strided_metadata operations. +// Note: Technically we could still fold the sizes and strides. +// +// CHECK-LABEL: func @extract_strided_metadata_of_cast_unranked +// CHECK-SAME: %[[ARG:.*]]: memref<*xi32>) +// +// CHECK: %[[CAST:.*]] = memref.cast %[[ARG]] : +// CHECK: %[[BASE:.*]], %[[OFFSET:.*]], %[[SIZES:.*]]:2, %[[STRIDES:.*]]:2 = memref.extract_strided_metadata %[[CAST]] +// +// CHECK: return %[[BASE]], %[[OFFSET]], %[[SIZES]]#0, %[[SIZES]]#1, %[[STRIDES]]#0, %[[STRIDES]]#1 +func.func @extract_strided_metadata_of_cast_unranked( + %arg : memref<*xi32>) + -> (memref<i32>, index, + index, index, + index, index) { + + %cast = + memref.cast %arg : + memref<*xi32> to + memref<?x?xi32, strided<[?, ?], offset: ?>> + + %base, %base_offset, %sizes:2, %strides:2 = + memref.extract_strided_metadata %cast:memref<?x?xi32, strided<[?, ?], offset: ?>> + -> memref<i32>, index, + index, index, + index, index + + return %base, %base_offset, + %sizes#0, %sizes#1, + %strides#0, %strides#1 : + memref<i32>, index, + index, index, + index, index +} + +// ----- + // CHECK-LABEL: func @reinterpret_noop // CHECK-SAME: (%[[ARG:.*]]: memref<2x3x4xf32>) // CHECK-NEXT: return %[[ARG]] diff --git a/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir b/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir index 1e6b011..18cdfb7 100644 --- a/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir +++ b/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir @@ -1378,133 +1378,6 @@ func.func @extract_strided_metadata_of_get_global_with_offset() // ----- -// Check that we simplify extract_strided_metadata of cast -// when the source of the cast is compatible with what -// `extract_strided_metadata`s accept. -// -// When we apply the transformation the resulting offset, sizes and strides -// should come straight from the inputs of the cast. -// Additionally the folder on extract_strided_metadata should propagate the -// static information. -// -// CHECK-LABEL: func @extract_strided_metadata_of_cast -// CHECK-SAME: %[[ARG:.*]]: memref<3x?xi32, strided<[4, ?], offset: ?>>) -// -// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index -// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : index -// CHECK: %[[BASE:.*]], %[[DYN_OFFSET:.*]], %[[DYN_SIZES:.*]]:2, %[[DYN_STRIDES:.*]]:2 = memref.extract_strided_metadata %[[ARG]] -// -// CHECK: return %[[BASE]], %[[DYN_OFFSET]], %[[C3]], %[[DYN_SIZES]]#1, %[[C4]], %[[DYN_STRIDES]]#1 -func.func @extract_strided_metadata_of_cast( - %arg : memref<3x?xi32, strided<[4, ?], offset:?>>) - -> (memref<i32>, index, - index, index, - index, index) { - - %cast = - memref.cast %arg : - memref<3x?xi32, strided<[4, ?], offset: ?>> to - memref<?x?xi32, strided<[?, ?], offset: ?>> - - %base, %base_offset, %sizes:2, %strides:2 = - memref.extract_strided_metadata %cast:memref<?x?xi32, strided<[?, ?], offset: ?>> - -> memref<i32>, index, - index, index, - index, index - - return %base, %base_offset, - %sizes#0, %sizes#1, - %strides#0, %strides#1 : - memref<i32>, index, - index, index, - index, index -} - -// ----- - -// Check that we simplify extract_strided_metadata of cast -// when the source of the cast is compatible with what -// `extract_strided_metadata`s accept. -// -// Same as extract_strided_metadata_of_cast but with constant sizes and strides -// in the destination type. -// -// CHECK-LABEL: func @extract_strided_metadata_of_cast_w_csts -// CHECK-SAME: %[[ARG:.*]]: memref<?x?xi32, strided<[?, ?], offset: ?>>) -// -// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : index -// CHECK-DAG: %[[C18:.*]] = arith.constant 18 : index -// CHECK-DAG: %[[C25:.*]] = arith.constant 25 : index -// CHECK: %[[BASE:.*]], %[[DYN_OFFSET:.*]], %[[DYN_SIZES:.*]]:2, %[[DYN_STRIDES:.*]]:2 = memref.extract_strided_metadata %[[ARG]] -// -// CHECK: return %[[BASE]], %[[C25]], %[[C4]], %[[DYN_SIZES]]#1, %[[DYN_STRIDES]]#0, %[[C18]] -func.func @extract_strided_metadata_of_cast_w_csts( - %arg : memref<?x?xi32, strided<[?, ?], offset:?>>) - -> (memref<i32>, index, - index, index, - index, index) { - - %cast = - memref.cast %arg : - memref<?x?xi32, strided<[?, ?], offset: ?>> to - memref<4x?xi32, strided<[?, 18], offset: 25>> - - %base, %base_offset, %sizes:2, %strides:2 = - memref.extract_strided_metadata %cast:memref<4x?xi32, strided<[?, 18], offset: 25>> - -> memref<i32>, index, - index, index, - index, index - - return %base, %base_offset, - %sizes#0, %sizes#1, - %strides#0, %strides#1 : - memref<i32>, index, - index, index, - index, index -} - -// ----- - -// Check that we don't simplify extract_strided_metadata of -// cast when the source of the cast is unranked. -// Unranked memrefs cannot feed into extract_strided_metadata operations. -// Note: Technically we could still fold the sizes and strides. -// -// CHECK-LABEL: func @extract_strided_metadata_of_cast_unranked -// CHECK-SAME: %[[ARG:.*]]: memref<*xi32>) -// -// CHECK: %[[CAST:.*]] = memref.cast %[[ARG]] : -// CHECK: %[[BASE:.*]], %[[OFFSET:.*]], %[[SIZES:.*]]:2, %[[STRIDES:.*]]:2 = memref.extract_strided_metadata %[[CAST]] -// -// CHECK: return %[[BASE]], %[[OFFSET]], %[[SIZES]]#0, %[[SIZES]]#1, %[[STRIDES]]#0, %[[STRIDES]]#1 -func.func @extract_strided_metadata_of_cast_unranked( - %arg : memref<*xi32>) - -> (memref<i32>, index, - index, index, - index, index) { - - %cast = - memref.cast %arg : - memref<*xi32> to - memref<?x?xi32, strided<[?, ?], offset: ?>> - - %base, %base_offset, %sizes:2, %strides:2 = - memref.extract_strided_metadata %cast:memref<?x?xi32, strided<[?, ?], offset: ?>> - -> memref<i32>, index, - index, index, - index, index - - return %base, %base_offset, - %sizes#0, %sizes#1, - %strides#0, %strides#1 : - memref<i32>, index, - index, index, - index, index -} - - -// ----- - memref.global "private" @dynamicShmem : memref<0xf16,3> // CHECK-LABEL: func @zero_sized_memred diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir index 8713689..77d18da 100644 --- a/mlir/test/Dialect/OpenACC/ops.mlir +++ b/mlir/test/Dialect/OpenACC/ops.mlir @@ -2200,3 +2200,46 @@ acc.private.recipe @privatization_memref_slice : memref<10x10xf32> init { acc.yield %result : memref<10x10xf32> } + +// ----- + +func.func @test_firstprivate_map(%arg0: memref<10xf32>) { + // Map the function argument using firstprivate_map to enable + // moving to accelerator but prevent any present counter updates. + %mapped = acc.firstprivate_map varPtr(%arg0 : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> + + acc.parallel { + // Allocate a local variable inside the parallel region to represent + // materialized privatization. + %local = memref.alloca() : memref<10xf32> + + // Initialize the local variable with the mapped firstprivate value + %c0 = arith.constant 0 : index + %c10 = arith.constant 10 : index + %c1 = arith.constant 1 : index + + scf.for %i = %c0 to %c10 step %c1 { + %val = memref.load %mapped[%i] : memref<10xf32> + memref.store %val, %local[%i] : memref<10xf32> + } + + acc.yield + } + + return +} + +// CHECK-LABEL: func @test_firstprivate_map +// CHECK-NEXT: %[[MAPPED:.*]] = acc.firstprivate_map varPtr(%{{.*}} : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> +// CHECK-NEXT: acc.parallel { +// CHECK-NEXT: %[[LOCAL:.*]] = memref.alloca() : memref<10xf32> +// CHECK-NEXT: %[[C0:.*]] = arith.constant 0 : index +// CHECK-NEXT: %[[C10:.*]] = arith.constant 10 : index +// CHECK-NEXT: %[[C1:.*]] = arith.constant 1 : index +// CHECK-NEXT: scf.for %{{.*}} = %[[C0]] to %[[C10]] step %[[C1]] { +// CHECK-NEXT: %{{.*}} = memref.load %[[MAPPED]][%{{.*}}] : memref<10xf32> +// CHECK-NEXT: memref.store %{{.*}}, %[[LOCAL]][%{{.*}}] : memref<10xf32> +// CHECK-NEXT: } +// CHECK-NEXT: acc.yield +// CHECK-NEXT: } +// CHECK-NEXT: return diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl b/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl index d2297d5..4b8a346 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl +++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl @@ -35,6 +35,7 @@ def math_test(name, hdrs = [], deps = [], **kwargs): "//libc:__support_fputil_manipulation_functions", "//libc:__support_fputil_nearest_integer_operations", "//libc:__support_fputil_normal_float", + "//libc:__support_macros_optimization", "//libc:__support_macros_properties_architectures", "//libc:__support_macros_properties_os", "//libc:__support_macros_properties_types", diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index b415a36..63d4aea 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -5697,10 +5697,17 @@ cc_binary( copts = llvm_copts, stamp = 0, deps = [ + ":AllTargetsAsmParsers", + ":AllTargetsCodeGens", + ":AllTargetsMCAs", ":Analysis", + ":CodeGen", ":Core", ":IRReader", + ":MC", ":Support", + ":Target", + ":TargetParser", ":config", ], ) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index b5f14ff..d528dae 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -9397,6 +9397,7 @@ cc_library( ":MemRefTransforms", ":NVGPUTransforms", ":OpenACCTransforms", + ":OpenMPTransforms", ":QuantTransforms", ":SCFTransforms", ":SPIRVTransforms", @@ -10162,6 +10163,42 @@ gentbl_cc_library( ) cc_library( + name = "OpenACCAnalysis", + srcs = glob( + [ + "lib/Dialect/OpenACC/Analysis/*.cpp", + ], + ), + hdrs = glob(["include/mlir/Dialect/OpenACC/Analysis/*.h"]), + includes = ["include"], + deps = [ + ":IR", + ":OpenACCDialect", + ":OpenACCUtils", + ":Pass", + ], +) + +cc_library( + name = "OpenACCUtils", + srcs = glob( + [ + "lib/Dialect/OpenACC/Utils/*.cpp", + ], + ), + includes = ["include"], + deps = [ + ":OpenACCDialect", + ":OpenACCOpsIncGen", + ":OpenACCPassIncGen", + ":OpenACCTypeInterfacesIncGen", + ":Support", + ":ViewLikeInterface", + "//llvm:Support", + ], +) + +cc_library( name = "OpenACCTransforms", srcs = glob( [ @@ -10314,6 +10351,40 @@ cc_library( ], ) +gentbl_cc_library( + name = "OpenMPPassIncGen", + tbl_outs = {"include/mlir/Dialect/OpenMP/Transforms/Passes.h.inc": [ + "-gen-pass-decls", + "-name=OpenMP", + ]}, + tblgen = ":mlir-tblgen", + td_file = "include/mlir/Dialect/OpenMP/Transforms/Passes.td", + deps = [":PassBaseTdFiles"], +) + +cc_library( + name = "OpenMPTransforms", + srcs = glob( + [ + "lib/Dialect/OpenMP/Transforms/*.cpp", + ], + ), + hdrs = glob(["include/mlir/Dialect/OpenMP/Transforms/*.h"]), + includes = ["include"], + deps = [ + ":Analysis", + ":FuncDialect", + ":IR", + ":LLVMDialect", + ":OpenMPDialect", + ":OpenMPPassIncGen", + ":Pass", + ":Support", + ":Transforms", + "//llvm:Support", + ], +) + cc_library( name = "OpenACCToSCF", srcs = glob([ diff --git a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel index 43613ad..be18ba5b 100644 --- a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel @@ -1029,6 +1029,7 @@ gentbl_filegroup( td_file = "mlir/dialects/ShardOps.td", deps = [ "//mlir:OpBaseTdFiles", + "//mlir:ShapeOpsTdFiles", "//mlir:ShardTdFiles", ], ) diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel index 56e4e70..778f0be 100644 --- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel @@ -963,6 +963,7 @@ cc_library( "//mlir:FuncDialect", "//mlir:IR", "//mlir:MemRefDialect", + "//mlir:OpenACCAnalysis", "//mlir:OpenACCDialect", "//mlir:Pass", "//mlir:Support", |