From d3bc9cc99b3d45e1fb8d65a57e308e899439fe26 Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Fri, 29 Mar 2024 15:45:06 +0100 Subject: [PATCH 0001/1690] [AArch64][GISEL] Regenerate select tests with inline register classes (#87013) Use inline register class syntax for select test file. --- .../CodeGen/AArch64/GlobalISel/select.mir | 333 ++++++++---------- 1 file changed, 143 insertions(+), 190 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select.mir index e207a31063ba..b3613f52c4ec 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/select.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/select.mir @@ -1,3 +1,4 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 # RUN: llc -O0 -mtriple=aarch64-apple-ios -run-pass=instruction-select -global-isel-abort=1 -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=IOS # RUN: llc -O0 -mtriple=aarch64-linux-gnu -relocation-model=pic -run-pass=instruction-select -global-isel-abort=1 -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=LINUX-PIC @@ -26,40 +27,35 @@ ... --- -# CHECK-LABEL: name: frame_index name: frame_index legalized: true regBankSelected: true - -# CHECK: registers: -# CHECK-NEXT: - { id: 0, class: gpr64sp, preferred-register: '' } -registers: - - { id: 0, class: gpr } - stack: - { id: 0, name: ptr0, offset: 0, size: 8, alignment: 8 } - -# CHECK: body: -# CHECK: %0:gpr64sp = ADDXri %stack.0.ptr0, 0, 0 body: | bb.0: - %0(p0) = G_FRAME_INDEX %stack.0.ptr0 + ; CHECK-LABEL: name: frame_index + ; CHECK: [[ADDXri:%[0-9]+]]:gpr64sp = ADDXri %stack.0.ptr0, 0, 0 + ; CHECK-NEXT: $x0 = COPY [[ADDXri]] + %0:gpr(p0) = G_FRAME_INDEX %stack.0.ptr0 $x0 = COPY %0(p0) ... --- --- -# CHECK-LABEL: name: ptr_mask name: ptr_mask legalized: true regBankSelected: true - -# CHECK: body: -# CHECK: %2:gpr64sp = ANDXri %0, 8060 body: | bb.0: liveins: $x0 + ; CHECK-LABEL: name: ptr_mask + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[ANDXri:%[0-9]+]]:gpr64sp = ANDXri [[COPY]], 8060 + ; CHECK-NEXT: $x0 = COPY [[ANDXri]] %0:gpr(p0) = COPY $x0 %const:gpr(s64) = G_CONSTANT i64 -8 %1:gpr(p0) = G_PTRMASK %0, %const @@ -68,200 +64,171 @@ body: | --- # Global defined in the same linkage unit so no GOT is needed -# CHECK-LABEL: name: global_local name: global_local legalized: true regBankSelected: true -registers: - - { id: 0, class: gpr } - -# CHECK: body: -# IOS: %0:gpr64common = MOVaddr target-flags(aarch64-page) @var_local, target-flags(aarch64-pageoff, aarch64-nc) @var_local -# LINUX-PIC: %0:gpr64common = LOADgot target-flags(aarch64-got) @var_local body: | bb.0: - %0(p0) = G_GLOBAL_VALUE @var_local + ; IOS-LABEL: name: global_local + ; IOS: [[MOVaddr:%[0-9]+]]:gpr64common = MOVaddr target-flags(aarch64-page) @var_local, target-flags(aarch64-pageoff, aarch64-nc) @var_local + ; IOS-NEXT: $x0 = COPY [[MOVaddr]] + ; + ; LINUX-PIC-LABEL: name: global_local + ; LINUX-PIC: [[LOADgot:%[0-9]+]]:gpr64common = LOADgot target-flags(aarch64-got) @var_local + ; LINUX-PIC-NEXT: $x0 = COPY [[LOADgot]] + %0:gpr(p0) = G_GLOBAL_VALUE @var_local $x0 = COPY %0(p0) ... --- -# CHECK-LABEL: name: global_got name: global_got legalized: true regBankSelected: true -registers: - - { id: 0, class: gpr } - -# CHECK: body: -# IOS: %0:gpr64common = LOADgot target-flags(aarch64-got) @var_got -# LINUX-PIC: %0:gpr64common = LOADgot target-flags(aarch64-got) @var_got body: | bb.0: - %0(p0) = G_GLOBAL_VALUE @var_got + ; CHECK-LABEL: name: global_got + ; CHECK: [[LOADgot:%[0-9]+]]:gpr64common = LOADgot target-flags(aarch64-got) @var_got + ; CHECK-NEXT: $x0 = COPY [[LOADgot]] + %0:gpr(p0) = G_GLOBAL_VALUE @var_got $x0 = COPY %0(p0) ... --- -# CHECK-LABEL: name: icmp name: icmp legalized: true regBankSelected: true - -# CHECK: registers: -# CHECK-NEXT: - { id: 0, class: gpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 1, class: gpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 2, class: gpr64, preferred-register: '' } -# CHECK-NEXT: - { id: 3, class: gpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 4, class: gpr64, preferred-register: '' } -# CHECK-NEXT: - { id: 5, class: gpr32, preferred-register: '' } -registers: - - { id: 0, class: gpr } - - { id: 1, class: gpr } - - { id: 2, class: gpr } - - { id: 3, class: gpr } - - { id: 4, class: gpr } - - { id: 5, class: gpr } - - { id: 6, class: gpr } - - { id: 7, class: gpr } - - { id: 8, class: gpr } - - { id: 9, class: gpr } - - { id: 10, class: gpr } - - { id: 11, class: gpr } - -# CHECK: body: -# CHECK: SUBSWrr %0, %0, implicit-def $nzcv -# CHECK: %1:gpr32 = CSINCWr $wzr, $wzr, 1, implicit $nzcv - -# CHECK: SUBSXrr %2, %2, implicit-def $nzcv -# CHECK: %3:gpr32 = CSINCWr $wzr, $wzr, 3, implicit $nzcv - -# CHECK: SUBSXrr %4, %4, implicit-def $nzcv -# CHECK: %5:gpr32 = CSINCWr $wzr, $wzr, 0, implicit $nzcv - body: | bb.0: liveins: $w0, $x0 - %0(s32) = COPY $w0 - %1(s32) = G_ICMP intpred(eq), %0, %0 - %6(s8) = G_TRUNC %1(s32) - %9(s32) = G_ANYEXT %6 + ; CHECK-LABEL: name: icmp + ; CHECK: liveins: $w0, $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0 + ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[COPY]], [[COPY]], implicit-def $nzcv + ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 1, implicit $nzcv + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32all = COPY [[CSINCWr]] + ; CHECK-NEXT: $w0 = COPY [[COPY1]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr [[COPY2]], [[COPY2]], implicit-def $nzcv + ; CHECK-NEXT: [[CSINCWr1:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 3, implicit $nzcv + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32all = COPY [[CSINCWr1]] + ; CHECK-NEXT: $w0 = COPY [[COPY3]] + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[SUBSXrr1:%[0-9]+]]:gpr64 = SUBSXrr [[COPY4]], [[COPY4]], implicit-def $nzcv + ; CHECK-NEXT: [[CSINCWr2:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 0, implicit $nzcv + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr32all = COPY [[CSINCWr2]] + ; CHECK-NEXT: $w0 = COPY [[COPY5]] + %0:gpr(s32) = COPY $w0 + %1:gpr(s32) = G_ICMP intpred(eq), %0, %0 + %6:gpr(s8) = G_TRUNC %1(s32) + %9:gpr(s32) = G_ANYEXT %6 $w0 = COPY %9(s32) - %2(s64) = COPY $x0 - %3(s32) = G_ICMP intpred(uge), %2, %2 - %7(s8) = G_TRUNC %3(s32) - %10(s32) = G_ANYEXT %7 + %2:gpr(s64) = COPY $x0 + %3:gpr(s32) = G_ICMP intpred(uge), %2, %2 + %7:gpr(s8) = G_TRUNC %3(s32) + %10:gpr(s32) = G_ANYEXT %7 $w0 = COPY %10(s32) - %4(p0) = COPY $x0 - %5(s32) = G_ICMP intpred(ne), %4, %4 - %8(s8) = G_TRUNC %5(s32) - %11(s32) = G_ANYEXT %8 + %4:gpr(p0) = COPY $x0 + %5:gpr(s32) = G_ICMP intpred(ne), %4, %4 + %8:gpr(s8) = G_TRUNC %5(s32) + %11:gpr(s32) = G_ANYEXT %8 $w0 = COPY %11(s32) ... --- -# CHECK-LABEL: name: fcmp name: fcmp legalized: true regBankSelected: true - -# CHECK: registers: -# CHECK-NEXT: - { id: 0, class: fpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 1, class: gpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 2, class: fpr64, preferred-register: '' } -# CHECK-NEXT: - { id: 3, class: gpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 4, class: gpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 5, class: gpr32, preferred-register: '' } -registers: - - { id: 0, class: fpr } - - { id: 1, class: gpr } - - { id: 2, class: fpr } - - { id: 3, class: gpr } - - { id: 4, class: gpr } - - { id: 5, class: gpr } - - { id: 6, class: gpr } - - { id: 7, class: gpr } - - { id: 8, class: fpr } - - { id: 9, class: gpr } - - { id: 10, class: fpr } - - { id: 11, class: gpr } - - { id: 12, class: gpr } - - { id: 13, class: gpr } - - { id: 14, class: gpr } - - { id: 15, class: gpr } - -# CHECK: body: -# CHECK: nofpexcept FCMPSrr %0, %0, implicit-def $nzcv -# CHECK: [[TST_MI:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 5, implicit $nzcv -# CHECK: [[TST_GT:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 13, implicit $nzcv -# CHECK: %1:gpr32 = ORRWrr [[TST_MI]], [[TST_GT]] - -# CHECK: nofpexcept FCMPDrr %2, %2, implicit-def $nzcv -# CHECK: %3:gpr32 = CSINCWr $wzr, $wzr, 4, implicit $nzcv - body: | bb.0: liveins: $w0, $x0 - %0(s32) = COPY $s0 - %1(s32) = G_FCMP floatpred(one), %0, %0 - %4(s8) = G_TRUNC %1(s32) - %6(s32) = G_ANYEXT %4 - $w0 = COPY %6(s32) + ; CHECK-LABEL: name: fcmp + ; CHECK: liveins: $w0, $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr32 = COPY $s0 + ; CHECK-NEXT: nofpexcept FCMPSrr [[COPY]], [[COPY]], implicit-def $nzcv, implicit $fpcr + ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 5, implicit $nzcv + ; CHECK-NEXT: [[CSINCWr1:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 13, implicit $nzcv + ; CHECK-NEXT: [[ORRWrr:%[0-9]+]]:gpr32 = ORRWrr [[CSINCWr]], [[CSINCWr1]] + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32all = COPY [[ORRWrr]] + ; CHECK-NEXT: $w0 = COPY [[COPY1]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:fpr64 = COPY $d0 + ; CHECK-NEXT: nofpexcept FCMPDrr [[COPY2]], [[COPY2]], implicit-def $nzcv, implicit $fpcr + ; CHECK-NEXT: [[CSINCWr2:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 4, implicit $nzcv + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32all = COPY [[CSINCWr2]] + ; CHECK-NEXT: $w0 = COPY [[COPY3]] + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:fpr32 = COPY $s0 + ; CHECK-NEXT: nofpexcept FCMPSrr [[COPY4]], [[COPY4]], implicit-def $nzcv, implicit $fpcr + ; CHECK-NEXT: [[CSINCWr3:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 15, implicit $nzcv + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr32all = COPY [[CSINCWr3]] + ; CHECK-NEXT: $w0 = COPY [[COPY5]] + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:fpr64 = COPY $d0 + ; CHECK-NEXT: nofpexcept FCMPDrr [[COPY6]], [[COPY6]], implicit-def $nzcv, implicit $fpcr + ; CHECK-NEXT: [[CSINCWr4:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 14, implicit $nzcv + ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr32all = COPY [[CSINCWr4]] + ; CHECK-NEXT: $w0 = COPY [[COPY7]] + %0:fpr(s32) = COPY $s0 + %1:gpr(s32) = G_FCMP floatpred(one), %0, %0 + %2:gpr(s8) = G_TRUNC %1(s32) + %3:gpr(s32) = G_ANYEXT %2 + $w0 = COPY %3(s32) - %2(s64) = COPY $d0 - %3(s32) = G_FCMP floatpred(uge), %2, %2 - %5(s8) = G_TRUNC %3(s32) - %7(s32) = G_ANYEXT %5 + %4:fpr(s64) = COPY $d0 + %5:gpr(s32) = G_FCMP floatpred(uge), %4, %4 + %6:gpr(s8) = G_TRUNC %5(s32) + %7:gpr(s32) = G_ANYEXT %6 $w0 = COPY %7(s32) - %8(s32) = COPY $s0 - %9(s32) = G_FCMP floatpred(true), %8, %8 - %12(s8) = G_TRUNC %9(s32) - %14(s32) = G_ANYEXT %12 - $w0 = COPY %14(s32) + %8:fpr(s32) = COPY $s0 + %9:gpr(s32) = G_FCMP floatpred(true), %8, %8 + %10:gpr(s8) = G_TRUNC %9(s32) + %11:gpr(s32) = G_ANYEXT %10 + $w0 = COPY %11(s32) - %10(s64) = COPY $d0 - %11(s32) = G_FCMP floatpred(false), %10, %10 - %13(s8) = G_TRUNC %11(s32) - %15(s32) = G_ANYEXT %13 + %12:fpr(s64) = COPY $d0 + %13:gpr(s32) = G_FCMP floatpred(false), %12, %12 + %14:gpr(s8) = G_TRUNC %13(s32) + %15:gpr(s32) = G_ANYEXT %14 $w0 = COPY %15(s32) ... --- -# CHECK-LABEL: name: phi name: phi legalized: true regBankSelected: true tracksRegLiveness: true - -# CHECK: registers: -# CHECK-NEXT: - { id: 0, class: fpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 1, class: gpr, preferred-register: '' } -# CHECK-NEXT: - { id: 2, class: fpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 3, class: gpr32, preferred-register: '' } -registers: - - { id: 0, class: fpr } - - { id: 1, class: gpr } - - { id: 2, class: fpr } - -# CHECK: body: -# CHECK: bb.1: -# CHECK: %2:fpr32 = PHI %0, %bb.0, %2, %bb.1 - body: | + ; CHECK-LABEL: name: phi + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: liveins: $s0, $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr32 = COPY $s0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[PHI:%[0-9]+]]:fpr32 = PHI [[COPY]], %bb.0, [[PHI]], %bb.1 + ; CHECK-NEXT: TBNZW [[COPY1]], 0, %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: $s0 = COPY [[PHI]] + ; CHECK-NEXT: RET_ReallyLR implicit $s0 bb.0: liveins: $s0, $w0 successors: %bb.1 - %0(s32) = COPY $s0 + %0:fpr(s32) = COPY $s0 %3:gpr(s32) = COPY $w0 bb.1: successors: %bb.1, %bb.2 - %2(s32) = PHI %0, %bb.0, %2, %bb.1 + %2:fpr(s32) = PHI %0, %bb.0, %2, %bb.1 G_BRCOND %3, %bb.1 bb.2: @@ -270,60 +237,46 @@ body: | ... --- -# CHECK-LABEL: name: select name: select legalized: true regBankSelected: true tracksRegLiveness: true - -# CHECK: registers: -# CHECK-NEXT: - { id: 0, class: gpr, preferred-register: '' } -# CHECK-NEXT: - { id: 1, class: gpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 2, class: gpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 3, class: gpr32, preferred-register: '' } -# CHECK-NEXT: - { id: 4, class: gpr64, preferred-register: '' } -# CHECK-NEXT: - { id: 5, class: gpr64, preferred-register: '' } -# CHECK-NEXT: - { id: 6, class: gpr64, preferred-register: '' } -# CHECK-NEXT: - { id: 7, class: gpr64, preferred-register: '' } -# CHECK-NEXT: - { id: 8, class: gpr64, preferred-register: '' } -# CHECK-NEXT: - { id: 9, class: gpr64, preferred-register: '' } -# CHECK-NEXT: - { id: 10, class: gpr32, preferred-register: '' } -registers: - - { id: 0, class: gpr } - - { id: 1, class: gpr } - - { id: 2, class: gpr } - - { id: 3, class: gpr } - - { id: 4, class: gpr } - - { id: 5, class: gpr } - - { id: 6, class: gpr } - - { id: 7, class: gpr } - - { id: 8, class: gpr } - - { id: 9, class: gpr } - -# CHECK: body: -# CHECK: ANDSWri %10, 0, implicit-def $nzcv -# CHECK: %3:gpr32 = CSELWr %1, %2, 1, implicit $nzcv -# CHECK: ANDSWri %10, 0, implicit-def $nzcv -# CHECK: %6:gpr64 = CSELXr %4, %5, 1, implicit $nzcv -# CHECK: ANDSWri %10, 0, implicit-def $nzcv -# CHECK: %9:gpr64 = CSELXr %7, %8, 1, implicit $nzcv body: | bb.0: liveins: $w0, $w1, $w2 + ; CHECK-LABEL: name: select + ; CHECK: liveins: $w0, $w1, $w2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2 + ; CHECK-NEXT: [[ANDSWri:%[0-9]+]]:gpr32 = ANDSWri [[COPY]], 0, implicit-def $nzcv + ; CHECK-NEXT: [[CSELWr:%[0-9]+]]:gpr32 = CSELWr [[COPY1]], [[COPY2]], 1, implicit $nzcv + ; CHECK-NEXT: $w0 = COPY [[CSELWr]] + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[ANDSWri1:%[0-9]+]]:gpr32 = ANDSWri [[COPY]], 0, implicit-def $nzcv + ; CHECK-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[COPY3]], [[COPY4]], 1, implicit $nzcv + ; CHECK-NEXT: $x0 = COPY [[CSELXr]] + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[ANDSWri2:%[0-9]+]]:gpr32 = ANDSWri [[COPY]], 0, implicit-def $nzcv + ; CHECK-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[COPY5]], [[COPY6]], 1, implicit $nzcv + ; CHECK-NEXT: $x0 = COPY [[CSELXr1]] %10:gpr(s32) = COPY $w0 - %1(s32) = COPY $w1 - %2(s32) = COPY $w2 - %3(s32) = G_SELECT %10, %1, %2 + %1:gpr(s32) = COPY $w1 + %2:gpr(s32) = COPY $w2 + %3:gpr(s32) = G_SELECT %10, %1, %2 $w0 = COPY %3(s32) - %4(s64) = COPY $x0 - %5(s64) = COPY $x1 - %6(s64) = G_SELECT %10, %4, %5 + %4:gpr(s64) = COPY $x0 + %5:gpr(s64) = COPY $x1 + %6:gpr(s64) = G_SELECT %10, %4, %5 $x0 = COPY %6(s64) - %7(p0) = COPY $x0 - %8(p0) = COPY $x1 - %9(p0) = G_SELECT %10, %7, %8 + %7:gpr(p0) = COPY $x0 + %8:gpr(p0) = COPY $x1 + %9:gpr(p0) = G_SELECT %10, %7, %8 $x0 = COPY %9(p0) ... -- GitLab From 0f6ed4c394fd8f843029f6919230bf8df8618529 Mon Sep 17 00:00:00 2001 From: Younan Zhang Date: Fri, 29 Mar 2024 23:28:54 +0800 Subject: [PATCH 0002/1690] [clang][Sema] Fix a CTAD regression after 42239d2e9 (#86914) The most recent declaration of a template as a friend can introduce a different template parameter depth compared to what we anticipate from a CTAD guide. Fixes https://github.com/llvm/llvm-project/issues/86769 --- clang/docs/ReleaseNotes.rst | 3 +++ clang/lib/Sema/SemaTemplate.cpp | 22 ++++++++++++++++- clang/test/SemaTemplate/concepts-friends.cpp | 26 ++++++++++++++++++++ clang/test/SemaTemplate/ctad.cpp | 2 +- 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 232de0d7d8bb..c193c20fae5c 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -357,6 +357,9 @@ Bug Fixes in This Version - Fixes an assertion failure on invalid code when trying to define member functions in lambdas. +- Fixed a regression in CTAD that a friend declaration that befriends itself may cause + incorrect constraint substitution. (#GH86769). + Bug Fixes to Compiler Builtins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index e575bb2df97f..9cd19d711af4 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -1836,7 +1836,27 @@ static TemplateParameterList *GetTemplateParameterList(TemplateDecl *TD) { // Make sure we get the template parameter list from the most // recent declaration, since that is the only one that is guaranteed to // have all the default template argument information. - return cast(TD->getMostRecentDecl())->getTemplateParameters(); + Decl *D = TD->getMostRecentDecl(); + // C++11 N3337 [temp.param]p12: + // A default template argument shall not be specified in a friend class + // template declaration. + // + // Skip past friend *declarations* because they are not supposed to contain + // default template arguments. Moreover, these declarations may introduce + // template parameters living in different template depths than the + // corresponding template parameters in TD, causing unmatched constraint + // substitution. + // + // FIXME: Diagnose such cases within a class template: + // template + // struct S { + // template friend struct C; + // }; + // template struct S; + while (D->getFriendObjectKind() != Decl::FriendObjectKind::FOK_None && + D->getPreviousDecl()) + D = D->getPreviousDecl(); + return cast(D)->getTemplateParameters(); } DeclResult Sema::CheckClassTemplate( diff --git a/clang/test/SemaTemplate/concepts-friends.cpp b/clang/test/SemaTemplate/concepts-friends.cpp index 255b0858917f..91b797034ed6 100644 --- a/clang/test/SemaTemplate/concepts-friends.cpp +++ b/clang/test/SemaTemplate/concepts-friends.cpp @@ -478,3 +478,29 @@ template class Foo { }; } // namespace FriendOfFriend + +namespace GH86769 { + +template +concept X = true; + +template struct Y { + Y(T) {} + template friend struct Y; + template friend struct Y; + template friend struct Y; +}; + +template +struct Z { + // FIXME: This is ill-formed per C++11 N3337 [temp.param]p12: + // A default template argument shall not be specified in a friend class + // template declaration. + template friend struct Y; +}; + +template struct Y; +template struct Z; +Y y(1); + +} diff --git a/clang/test/SemaTemplate/ctad.cpp b/clang/test/SemaTemplate/ctad.cpp index 388ed7d4cced..ec144d4f44ba 100644 --- a/clang/test/SemaTemplate/ctad.cpp +++ b/clang/test/SemaTemplate/ctad.cpp @@ -53,4 +53,4 @@ X x; template struct Y { Y(T); }; template struct Y ; Y y(1); -}; +} -- GitLab From a4de589d117a4fd52554da3c61ae6eb26c90a0c8 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Fri, 29 Mar 2024 11:29:22 -0400 Subject: [PATCH 0003/1690] [InstallAPI] Add support for parsing dSYMs (#86852) InstallAPI does not directly look at object files in the dylib for verification. To help diagnose violations where a declaration is undiscovered in headers, parse the dSYM and look up the source location for symbols. Emitting out the source location with a diagnostic is enough for some IDE's (e.g. Xcode) to have them map back to editable source files. --- .../include/clang/InstallAPI/DylibVerifier.h | 20 +++- clang/include/clang/InstallAPI/MachO.h | 1 + clang/lib/InstallAPI/CMakeLists.txt | 1 + clang/lib/InstallAPI/DylibVerifier.cpp | 71 ++++++++--- clang/test/InstallAPI/diagnostics-dsym.test | 43 +++++++ .../tools/clang-installapi/InstallAPIOpts.td | 2 + clang/tools/clang-installapi/Options.cpp | 6 +- clang/tools/clang-installapi/Options.h | 3 + llvm/include/llvm/TextAPI/DylibReader.h | 9 ++ llvm/include/llvm/TextAPI/Record.h | 17 +++ llvm/lib/TextAPI/BinaryReader/CMakeLists.txt | 1 + llvm/lib/TextAPI/BinaryReader/DylibReader.cpp | 111 +++++++++++++++++- 12 files changed, 263 insertions(+), 22 deletions(-) create mode 100644 clang/test/InstallAPI/diagnostics-dsym.test diff --git a/clang/include/clang/InstallAPI/DylibVerifier.h b/clang/include/clang/InstallAPI/DylibVerifier.h index 49de24763f1f..22cdc234486c 100644 --- a/clang/include/clang/InstallAPI/DylibVerifier.h +++ b/clang/include/clang/InstallAPI/DylibVerifier.h @@ -31,6 +31,7 @@ enum class VerificationMode { class DylibVerifier : llvm::MachO::RecordVisitor { private: struct SymbolContext; + struct DWARFContext; public: enum class Result { NoVerify, Ignore, Valid, Invalid }; @@ -54,7 +55,7 @@ public: DiagnosticsEngine *Diag = nullptr; // Handle diagnostics reporting for target level violations. - void emitDiag(llvm::function_ref Report); + void emitDiag(llvm::function_ref Report, RecordLoc *Loc = nullptr); VerifierContext() = default; VerifierContext(DiagnosticsEngine *Diag) : Diag(Diag) {} @@ -63,9 +64,10 @@ public: DylibVerifier() = default; DylibVerifier(llvm::MachO::Records &&Dylib, DiagnosticsEngine *Diag, - VerificationMode Mode, bool Demangle) + VerificationMode Mode, bool Demangle, StringRef DSYMPath) : Dylib(std::move(Dylib)), Mode(Mode), Demangle(Demangle), - Exports(std::make_unique()), Ctx(VerifierContext{Diag}) {} + DSYMPath(DSYMPath), Exports(std::make_unique()), + Ctx(VerifierContext{Diag}) {} Result verify(GlobalRecord *R, const FrontendAttrs *FA); Result verify(ObjCInterfaceRecord *R, const FrontendAttrs *FA); @@ -143,6 +145,12 @@ private: std::string getAnnotatedName(const Record *R, SymbolContext &SymCtx, bool ValidSourceLoc = true); + /// Extract source location for symbol implementations. + /// As this is a relatively expensive operation, it is only used + /// when there is a violation to report and there is not a known declaration + /// in the interface. + void accumulateSrcLocForDylibSymbols(); + // Symbols in dylib. llvm::MachO::Records Dylib; @@ -152,11 +160,17 @@ private: // Attempt to demangle when reporting violations. bool Demangle = false; + // File path to DSYM file. + StringRef DSYMPath; + // Valid symbols in final text file. std::unique_ptr Exports = std::make_unique(); // Track current state of verification while traversing AST. VerifierContext Ctx; + + // Track DWARF provided source location for dylibs. + DWARFContext *DWARFCtx = nullptr; }; } // namespace installapi diff --git a/clang/include/clang/InstallAPI/MachO.h b/clang/include/clang/InstallAPI/MachO.h index 4961c596fd68..827220dbf39f 100644 --- a/clang/include/clang/InstallAPI/MachO.h +++ b/clang/include/clang/InstallAPI/MachO.h @@ -34,6 +34,7 @@ using ObjCCategoryRecord = llvm::MachO::ObjCCategoryRecord; using ObjCIVarRecord = llvm::MachO::ObjCIVarRecord; using ObjCIFSymbolKind = llvm::MachO::ObjCIFSymbolKind; using Records = llvm::MachO::Records; +using RecordLoc = llvm::MachO::RecordLoc; using RecordsSlice = llvm::MachO::RecordsSlice; using BinaryAttrs = llvm::MachO::RecordsSlice::BinaryAttrs; using SymbolSet = llvm::MachO::SymbolSet; diff --git a/clang/lib/InstallAPI/CMakeLists.txt b/clang/lib/InstallAPI/CMakeLists.txt index 894db699578f..e0bc8d969ecb 100644 --- a/clang/lib/InstallAPI/CMakeLists.txt +++ b/clang/lib/InstallAPI/CMakeLists.txt @@ -1,6 +1,7 @@ set(LLVM_LINK_COMPONENTS Support TextAPI + TextAPIBinaryReader Demangle Core ) diff --git a/clang/lib/InstallAPI/DylibVerifier.cpp b/clang/lib/InstallAPI/DylibVerifier.cpp index ba25e4183a9b..c0eda1d81b9b 100644 --- a/clang/lib/InstallAPI/DylibVerifier.cpp +++ b/clang/lib/InstallAPI/DylibVerifier.cpp @@ -10,6 +10,7 @@ #include "clang/InstallAPI/FrontendRecords.h" #include "clang/InstallAPI/InstallAPIDiagnostic.h" #include "llvm/Demangle/Demangle.h" +#include "llvm/TextAPI/DylibReader.h" using namespace llvm::MachO; @@ -35,6 +36,14 @@ struct DylibVerifier::SymbolContext { bool Inlined = false; }; +struct DylibVerifier::DWARFContext { + // Track whether DSYM parsing has already been attempted to avoid re-parsing. + bool ParsedDSYM{false}; + + // Lookup table for source locations by symbol name. + DylibReader::SymbolToSourceLocMap SourceLocs{}; +}; + static bool isCppMangled(StringRef Name) { // InstallAPI currently only supports itanium manglings. return (Name.starts_with("_Z") || Name.starts_with("__Z") || @@ -511,14 +520,16 @@ DylibVerifier::Result DylibVerifier::verify(GlobalRecord *R, return verifyImpl(R, SymCtx); } -void DylibVerifier::VerifierContext::emitDiag( - llvm::function_ref Report) { +void DylibVerifier::VerifierContext::emitDiag(llvm::function_ref Report, + RecordLoc *Loc) { if (!DiscoveredFirstError) { Diag->Report(diag::warn_target) << (PrintArch ? getArchitectureName(Target.Arch) : getTargetTripleName(Target)); DiscoveredFirstError = true; } + if (Loc && Loc->isValid()) + llvm::errs() << Loc->File << ":" << Loc->Line << ":" << 0 << ": "; Report(); } @@ -561,26 +572,36 @@ void DylibVerifier::visitSymbolInDylib(const Record &R, SymbolContext &SymCtx) { return; } - // All checks at this point classify as some kind of violation that should be - // reported. + const bool IsLinkerSymbol = SymbolName.starts_with("$ld$"); + + // All checks at this point classify as some kind of violation. + // The different verification modes dictate whether they are reported to the + // user. + if (IsLinkerSymbol || (Mode > VerificationMode::ErrorsOnly)) + accumulateSrcLocForDylibSymbols(); + RecordLoc Loc = DWARFCtx->SourceLocs.lookup(SymCtx.SymbolName); // Regardless of verification mode, error out on mismatched special linker // symbols. - if (SymbolName.starts_with("$ld$")) { - Ctx.emitDiag([&]() { - Ctx.Diag->Report(diag::err_header_symbol_missing) - << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); - }); + if (IsLinkerSymbol) { + Ctx.emitDiag( + [&]() { + Ctx.Diag->Report(diag::err_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, Loc.isValid()); + }, + &Loc); updateState(Result::Invalid); return; } // Missing declarations for exported symbols are hard errors on Pedantic mode. if (Mode == VerificationMode::Pedantic) { - Ctx.emitDiag([&]() { - Ctx.Diag->Report(diag::err_header_symbol_missing) - << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); - }); + Ctx.emitDiag( + [&]() { + Ctx.Diag->Report(diag::err_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, Loc.isValid()); + }, + &Loc); updateState(Result::Invalid); return; } @@ -588,10 +609,12 @@ void DylibVerifier::visitSymbolInDylib(const Record &R, SymbolContext &SymCtx) { // Missing declarations for exported symbols are warnings on ErrorsAndWarnings // mode. if (Mode == VerificationMode::ErrorsAndWarnings) { - Ctx.emitDiag([&]() { - Ctx.Diag->Report(diag::warn_header_symbol_missing) - << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); - }); + Ctx.emitDiag( + [&]() { + Ctx.Diag->Report(diag::warn_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, Loc.isValid()); + }, + &Loc); updateState(Result::Ignore); return; } @@ -622,6 +645,18 @@ void DylibVerifier::visitObjCIVar(const ObjCIVarRecord &R, visitSymbolInDylib(R, SymCtx); } +void DylibVerifier::accumulateSrcLocForDylibSymbols() { + if (DSYMPath.empty()) + return; + + assert(DWARFCtx != nullptr && "Expected an initialized DWARFContext"); + if (DWARFCtx->ParsedDSYM) + return; + DWARFCtx->ParsedDSYM = true; + DWARFCtx->SourceLocs = + DylibReader::accumulateSourceLocFromDSYM(DSYMPath, Ctx.Target); +} + void DylibVerifier::visitObjCInterface(const ObjCInterfaceRecord &R) { if (R.isVerified()) return; @@ -655,6 +690,8 @@ DylibVerifier::Result DylibVerifier::verifyRemainingSymbols() { return Result::NoVerify; assert(!Dylib.empty() && "No binary to verify against"); + DWARFContext DWARFInfo; + DWARFCtx = &DWARFInfo; Ctx.DiscoveredFirstError = false; Ctx.PrintArch = true; for (std::shared_ptr Slice : Dylib) { diff --git a/clang/test/InstallAPI/diagnostics-dsym.test b/clang/test/InstallAPI/diagnostics-dsym.test new file mode 100644 index 000000000000..ee2c8b32df29 --- /dev/null +++ b/clang/test/InstallAPI/diagnostics-dsym.test @@ -0,0 +1,43 @@ +; REQUIRES: system-darwin + +; RUN: rm -rf %t +; RUN: split-file %s %t + +// Build a simple dylib with debug info. +; RUN: %clang --target=arm64-apple-macos13 -g -dynamiclib %t/foo.c \ +; RUN: -current_version 1 -compatibility_version 1 -L%t/usr/lib \ +; RUN: -save-temps \ +; RUN: -o %t/foo.dylib -install_name %t/foo.dylib +; RUN: dsymutil %t/foo.dylib -o %t/foo.dSYM + +; RUN: not clang-installapi -x c++ --target=arm64-apple-macos13 \ +; RUN: -install_name %t/foo.dylib \ +; RUN: -current_version 1 -compatibility_version 1 \ +; RUN: -o %t/output.tbd \ +; RUN: --verify-against=%t/foo.dylib --dsym=%t/foo.dSYM \ +; RUN: --verify-mode=Pedantic 2>&1 | FileCheck %s + +; CHECK: violations found for arm64 +; CHECK: foo.c:5:0: error: no declaration found for exported symbol 'bar' in dynamic library +; CHECK: foo.c:1:0: error: no declaration found for exported symbol 'foo' in dynamic library + +;--- foo.c +int foo(void) { + return 1; +} +extern char bar; +char bar = 'a'; + +;--- usr/lib/libSystem.tbd +{ + "main_library": { + "install_names": [ + {"name": "/usr/lib/libSystem.B.dylib"} + ], + "target_info": [ + {"target": "arm64-macos"} + ] + }, + "tapi_tbd_version": 5 +} + diff --git a/clang/tools/clang-installapi/InstallAPIOpts.td b/clang/tools/clang-installapi/InstallAPIOpts.td index 71532c9cf24d..010f2507a1d1 100644 --- a/clang/tools/clang-installapi/InstallAPIOpts.td +++ b/clang/tools/clang-installapi/InstallAPIOpts.td @@ -29,6 +29,8 @@ def verify_mode_EQ : Joined<["--"], "verify-mode=">, HelpText<"Specify the severity and extend of the validation. Valid modes are ErrorsOnly, ErrorsAndWarnings, and Pedantic.">; def demangle : Flag<["--", "-"], "demangle">, HelpText<"Demangle symbols when printing warnings and errors">; +def dsym: Joined<["--"], "dsym=">, + MetaVarName<"">, HelpText<"Specify dSYM path for enriched diagnostics.">; // Additional input options. def extra_project_header : Separate<["-"], "extra-project-header">, diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp index 8e4a1b019fd8..c4f39b7c8417 100644 --- a/clang/tools/clang-installapi/Options.cpp +++ b/clang/tools/clang-installapi/Options.cpp @@ -241,6 +241,9 @@ Options::processAndFilterOutInstallAPIOptions(ArrayRef Args) { if (const Arg *A = ParsedArgs.getLastArg(OPT_verify_against)) DriverOpts.DylibToVerify = A->getValue(); + if (const Arg *A = ParsedArgs.getLastArg(OPT_dsym)) + DriverOpts.DSYMPath = A->getValue(); + // Handle exclude & extra header directories or files. auto handleAdditionalInputArgs = [&](PathSeq &Headers, clang::installapi::ID OptID) { @@ -522,7 +525,8 @@ InstallAPIContext Options::createContext() { } Ctx.Verifier = std::make_unique( - std::move(*Slices), Diags, DriverOpts.VerifyMode, DriverOpts.Demangle); + std::move(*Slices), Diags, DriverOpts.VerifyMode, DriverOpts.Demangle, + DriverOpts.DSYMPath); return Ctx; } diff --git a/clang/tools/clang-installapi/Options.h b/clang/tools/clang-installapi/Options.h index 3671e4c8274b..82e04b49d125 100644 --- a/clang/tools/clang-installapi/Options.h +++ b/clang/tools/clang-installapi/Options.h @@ -67,6 +67,9 @@ struct DriverOptions { /// \brief Output path. std::string OutputPath; + /// \brief DSYM path. + std::string DSYMPath; + /// \brief File encoding to print. FileType OutFT = FileType::TBD_V5; diff --git a/llvm/include/llvm/TextAPI/DylibReader.h b/llvm/include/llvm/TextAPI/DylibReader.h index b556fbf6832a..6861d3cb1591 100644 --- a/llvm/include/llvm/TextAPI/DylibReader.h +++ b/llvm/include/llvm/TextAPI/DylibReader.h @@ -13,6 +13,7 @@ #ifndef LLVM_TEXTAPI_DYLIBREADER_H #define LLVM_TEXTAPI_DYLIBREADER_H +#include "llvm/ADT/StringMap.h" #include "llvm/Support/Error.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/TextAPI/ArchitectureSet.h" @@ -43,6 +44,14 @@ Expected readFile(MemoryBufferRef Buffer, const ParseOption &Opt); /// \param Buffer Data that points to dylib. Expected> get(MemoryBufferRef Buffer); +using SymbolToSourceLocMap = llvm::StringMap; +/// Get the source location for each symbol from dylib. +/// +/// \param DSYM Path to DSYM file. +/// \param T Requested target slice for dylib. +SymbolToSourceLocMap accumulateSourceLocFromDSYM(const StringRef DSYM, + const Target &T); + } // namespace llvm::MachO::DylibReader #endif // LLVM_TEXTAPI_DYLIBREADER_H diff --git a/llvm/include/llvm/TextAPI/Record.h b/llvm/include/llvm/TextAPI/Record.h index ef152ce43387..7d721988ec3d 100644 --- a/llvm/include/llvm/TextAPI/Record.h +++ b/llvm/include/llvm/TextAPI/Record.h @@ -27,6 +27,23 @@ LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); class RecordsSlice; +// Defines lightweight source location for records. +struct RecordLoc { + RecordLoc() = default; + RecordLoc(std::string File, unsigned Line) + : File(std::move(File)), Line(Line) {} + + /// Whether there is source location tied to the RecordLoc object. + bool isValid() const { return !File.empty(); } + + bool operator==(const RecordLoc &O) const { + return std::tie(File, Line) == std::tie(O.File, O.Line); + } + + const std::string File; + const unsigned Line = 0; +}; + // Defines a list of linkage types. enum class RecordLinkage : uint8_t { // Unknown linkage. diff --git a/llvm/lib/TextAPI/BinaryReader/CMakeLists.txt b/llvm/lib/TextAPI/BinaryReader/CMakeLists.txt index cbdf7b2c9696..c4535310d91c 100644 --- a/llvm/lib/TextAPI/BinaryReader/CMakeLists.txt +++ b/llvm/lib/TextAPI/BinaryReader/CMakeLists.txt @@ -2,6 +2,7 @@ add_llvm_component_library(LLVMTextAPIBinaryReader DylibReader.cpp LINK_COMPONENTS + DebugInfoDWARF Support Object TextAPI diff --git a/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp b/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp index 2e36d4a8b98c..f92a2d19a63f 100644 --- a/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp +++ b/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp @@ -12,7 +12,8 @@ #include "llvm/TextAPI/DylibReader.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringMap.h" +#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/Object/Binary.h" #include "llvm/Object/MachOUniversal.h" #include "llvm/Support/Endian.h" @@ -432,3 +433,111 @@ DylibReader::get(MemoryBufferRef Buffer) { return convertToInterfaceFile(*SlicesOrErr); } + +static void DWARFErrorHandler(Error Err) { /**/ } + +static SymbolToSourceLocMap +accumulateLocs(MachOObjectFile &Obj, + const std::unique_ptr &DiCtx) { + SymbolToSourceLocMap LocMap; + for (const auto &Symbol : Obj.symbols()) { + Expected FlagsOrErr = Symbol.getFlags(); + if (!FlagsOrErr) { + consumeError(FlagsOrErr.takeError()); + continue; + } + + if (!(*FlagsOrErr & SymbolRef::SF_Exported)) + continue; + + Expected AddressOrErr = Symbol.getAddress(); + if (!AddressOrErr) { + consumeError(AddressOrErr.takeError()); + continue; + } + const uint64_t Address = *AddressOrErr; + + auto TypeOrErr = Symbol.getType(); + if (!TypeOrErr) { + consumeError(TypeOrErr.takeError()); + continue; + } + const bool IsCode = (*TypeOrErr & SymbolRef::ST_Function); + + auto *DWARFCU = IsCode ? DiCtx->getCompileUnitForCodeAddress(Address) + : DiCtx->getCompileUnitForDataAddress(Address); + if (!DWARFCU) + continue; + + const DWARFDie &DIE = IsCode ? DWARFCU->getSubroutineForAddress(Address) + : DWARFCU->getVariableForAddress(Address); + const std::string File = DIE.getDeclFile( + llvm::DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath); + const uint64_t Line = DIE.getDeclLine(); + + auto NameOrErr = Symbol.getName(); + if (!NameOrErr) { + consumeError(NameOrErr.takeError()); + continue; + } + auto Name = *NameOrErr; + auto Sym = parseSymbol(Name); + + if (!File.empty() && Line != 0) + LocMap.insert({Sym.Name.str(), RecordLoc(File, Line)}); + } + + return LocMap; +} + +SymbolToSourceLocMap +DylibReader::accumulateSourceLocFromDSYM(const StringRef DSYM, + const Target &T) { + // Find sidecar file. + auto DSYMsOrErr = MachOObjectFile::findDsymObjectMembers(DSYM); + if (!DSYMsOrErr) { + consumeError(DSYMsOrErr.takeError()); + return SymbolToSourceLocMap(); + } + if (DSYMsOrErr->empty()) + return SymbolToSourceLocMap(); + + const StringRef Path = DSYMsOrErr->front(); + auto BufOrErr = MemoryBuffer::getFile(Path); + if (auto Err = BufOrErr.getError()) + return SymbolToSourceLocMap(); + + auto BinOrErr = createBinary(*BufOrErr.get()); + if (!BinOrErr) { + consumeError(BinOrErr.takeError()); + return SymbolToSourceLocMap(); + } + // Handle single arch. + if (auto *Single = dyn_cast(BinOrErr->get())) { + auto DiCtx = DWARFContext::create( + *Single, DWARFContext::ProcessDebugRelocations::Process, nullptr, "", + DWARFErrorHandler, DWARFErrorHandler); + + return accumulateLocs(*Single, DiCtx); + } + // Handle universal companion file. + if (auto *Fat = dyn_cast(BinOrErr->get())) { + auto ObjForArch = Fat->getObjectForArch(getArchitectureName(T.Arch)); + if (!ObjForArch) { + consumeError(ObjForArch.takeError()); + return SymbolToSourceLocMap(); + } + auto MachOOrErr = ObjForArch->getAsObjectFile(); + if (!MachOOrErr) { + consumeError(MachOOrErr.takeError()); + return SymbolToSourceLocMap(); + } + auto &Obj = **MachOOrErr; + auto DiCtx = DWARFContext::create( + Obj, DWARFContext::ProcessDebugRelocations::Process, nullptr, "", + DWARFErrorHandler, DWARFErrorHandler); + + return accumulateLocs(Obj, DiCtx); + } + return SymbolToSourceLocMap(); +} -- GitLab From 85a5f64c70fe34a08481fde2cfedb6adf84c9b47 Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Fri, 29 Mar 2024 16:14:09 +0000 Subject: [PATCH 0004/1690] [bazel] Add missing deps after a4de589d117a4fd52554da3c61ae6eb26c90a0c8 --- utils/bazel/llvm-project-overlay/clang/BUILD.bazel | 1 + utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 1 + 2 files changed, 2 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel index c01986815afe..1bf6bee10952 100644 --- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel @@ -2056,6 +2056,7 @@ cc_library( "//llvm:Demangle", "//llvm:Support", "//llvm:TextAPI", + "//llvm:TextAPIBinaryReader", ], ) diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index 15e477351fe3..94871c8be6e5 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -1011,6 +1011,7 @@ cc_library( hdrs = ["include/llvm/TextAPI/DylibReader.h"], copts = llvm_copts, deps = [ + ":DebugInfoDWARF", ":Object", ":Support", ":TargetParser", -- GitLab From c09b6fac12b0299841bf1bf04974712963736db5 Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Fri, 29 Mar 2024 11:20:09 -0500 Subject: [PATCH 0005/1690] [bazel] Add support for LLVM_TOOL_LLVM_DRIVER_BUILD (#86879) This adds the bazel equivalent of the `llvm` binary produced by `LLVM_TOOL_LLVM_DRIVER_BUILD` in cmake. For the initial commit, this only includes `llvm-ar`, `llvm-nm`, and `llvm-size`. The rest are trivial to add in a followup commit, following the same pattern as here. By default it will include everything that supports the llvm-driver model, but it can be reduced to only build a subset, e.g. this will build only nm and size: ``` $ bazel build \ --@llvm-project//llvm:driver-tools=llvm-nm,llvm-size \ @llvm-project//llvm:llvm ``` --- .../llvm-project-overlay/llvm/BUILD.bazel | 99 +++++----- .../llvm-project-overlay/llvm/driver.bzl | 182 ++++++++++++++++++ 2 files changed, 236 insertions(+), 45 deletions(-) create mode 100644 utils/bazel/llvm-project-overlay/llvm/driver.bzl diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index 94871c8be6e5..9cfcb7d3838e 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -7,6 +7,7 @@ load("@bazel_skylib//rules:expand_template.bzl", "expand_template") load("//mlir:tblgen.bzl", "td_library") load(":binary_alias.bzl", "binary_alias") load(":config.bzl", "llvm_config_defines") +load(":driver.bzl", "generate_driver_selects", "generate_driver_tools_def", "llvm_driver_cc_binary", "select_driver_tools") load(":enum_targets_gen.bzl", "enum_targets_gen") load(":targets.bzl", "llvm_targets") load(":tblgen.bzl", "gentbl") @@ -605,6 +606,32 @@ cc_library( ], ) +# Command line flag to control which tools get included in the llvm driver binary. +# The macro also generates config_setting targets used by select_driver_tools(). +generate_driver_selects(name = "driver-tools") + +generate_driver_tools_def( + name = "gen_llvm_driver_tools_def", + out = "LLVMDriverTools.def", + driver_tools = select_driver_tools(":driver-tools"), +) + +# Workaround inability to put `.def` files into `srcs` with a library +cc_library( + name = "llvm_driver_tools_def_lib", + includes = ["."], + textual_hdrs = ["LLVMDriverTools.def"], +) + +cc_binary( + name = "llvm", + srcs = glob(["tools/llvm-driver/*.cpp"]), + deps = [ + ":Support", + ":llvm_driver_tools_def_lib", + ] + select_driver_tools(":driver-tools"), +) + cc_binary( name = "llvm-min-tblgen", srcs = [ @@ -3305,22 +3332,10 @@ cc_binary( ], ) -expand_template( - name = "ar_main", - out = "llvm-ar-driver.cpp", - substitutions = { - "@TOOL_NAME@": "llvm_ar", - }, - template = "cmake/modules/llvm-driver-template.cpp.in", -) - -cc_binary( - name = "llvm-ar", - srcs = glob([ - "tools/llvm-ar/*.cpp", - ]) + ["llvm-ar-driver.cpp"], +cc_library( + name = "llvm-ar-lib", + srcs = glob(["tools/llvm-ar/*.cpp"]), copts = llvm_copts, - stamp = 0, deps = [ ":AllTargetsAsmParsers", ":AllTargetsCodeGens", @@ -3334,6 +3349,12 @@ cc_binary( ], ) +llvm_driver_cc_binary( + name = "llvm-ar", + stamp = 0, + deps = [":llvm-ar-lib"], +) + # We need to run llvm-ar with different basenames to make it run with # different behavior. binary_alias( @@ -4151,22 +4172,10 @@ gentbl( td_srcs = ["include/llvm/Option/OptParser.td"], ) -expand_template( - name = "nm_main", - out = "llvm-nm-driver.cpp", - substitutions = { - "@TOOL_NAME@": "llvm_nm", - }, - template = "cmake/modules/llvm-driver-template.cpp.in", -) - -cc_binary( - name = "llvm-nm", - srcs = glob([ - "tools/llvm-nm/*.cpp", - ]) + ["llvm-nm-driver.cpp"], +cc_library( + name = "llvm-nm-lib", + srcs = glob(["tools/llvm-nm/*.cpp"]), copts = llvm_copts, - stamp = 0, deps = [ ":AllTargetsAsmParsers", ":AllTargetsCodeGens", @@ -4183,6 +4192,12 @@ cc_binary( ], ) +llvm_driver_cc_binary( + name = "llvm-nm", + stamp = 0, + deps = [":llvm-nm-lib"], +) + gentbl( name = "llvm-objcopy-opts", strip_include_prefix = "tools/llvm-objcopy", @@ -4634,22 +4649,10 @@ gentbl( td_srcs = ["include/llvm/Option/OptParser.td"], ) -expand_template( - name = "size_main", - out = "llvm-size-driver.cpp", - substitutions = { - "@TOOL_NAME@": "llvm_size", - }, - template = "cmake/modules/llvm-driver-template.cpp.in", -) - -cc_binary( - name = "llvm-size", - srcs = glob([ - "tools/llvm-size/*.cpp", - ]) + ["llvm-size-driver.cpp"], +cc_library( + name = "llvm-size-lib", + srcs = glob(["tools/llvm-size/*.cpp"]), copts = llvm_copts, - stamp = 0, deps = [ ":Object", ":Option", @@ -4658,6 +4661,12 @@ cc_binary( ], ) +llvm_driver_cc_binary( + name = "llvm-size", + stamp = 0, + deps = [":llvm-size-lib"], +) + cc_binary( name = "llvm-split", srcs = glob([ diff --git a/utils/bazel/llvm-project-overlay/llvm/driver.bzl b/utils/bazel/llvm-project-overlay/llvm/driver.bzl new file mode 100644 index 000000000000..bd0d26d64f48 --- /dev/null +++ b/utils/bazel/llvm-project-overlay/llvm/driver.bzl @@ -0,0 +1,182 @@ +# This file is licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +"""Configuration for the llvm-driver tool.""" + +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@bazel_skylib//rules:expand_template.bzl", "expand_template") + +# Mapping from every tool to the cc_library that implements the tool's entrypoint. +# TODO: uncomment the remaining targets after splitting them +# into separate library/binary targets. +_TOOLS = { + # "clang-scan-deps": "//clang:clang-scan-deps-lib", + # "clang": "//clang:clang-driver", + # "dsymutil": "//llvm:dsymutil-lib", + # "lld": "//lld:lld-lib", + "llvm-ar": "//llvm:llvm-ar-lib", + # "llvm-cxxfilt": "//llvm:llvm-cxxfilt-lib", + # "llvm-dwp": "//llvm:llvm-dwp-lib", + # "llvm-gsymutil": "//llvm:llvm-gsymutil-lib", + # "llvm-ifs": "//llvm:llvm-ifs-lib", + # "llvm-libtool-darwin": "//llvm:llvm-libtool-darwin-lib", + # "llvm-lipo": "//llvm:llvm-lipo-lib", + # "llvm-ml": "//llvm:llvm-ml-lib", + # "llvm-mt": "//llvm:llvm-mt-lib", + "llvm-nm": "//llvm:llvm-nm-lib", + # "llvm-objcopy": "//llvm:llvm-objcopy-lib", + # "llvm-objdump": "//llvm:llvm-objdump-lib", + # "llvm-profdata": "//llvm:llvm-profdata-lib", + # "llvm-rc": "//llvm:llvm-rc-lib", + # "llvm-readobj": "//llvm:llvm-readobj-lib", + "llvm-size": "//llvm:llvm-size-lib", + # "llvm-symbolizer": "//llvm:llvm-symbolizer-lib", + # "sancov": "//llvm:sancov-lib", +} + +# Tools automatically get their own name as an alias, but there may be additional +# aliases for a given tool. +_EXTRA_ALIASES = { + "clang": ["clang++", "clang-cl", "clang-cpp"], + "lld": ["lld-link", "ld.lld", "ld64.lld", "wasm-ld"], + "llvm-ar": ["ranlib", "lib", "dlltool"], + "llvm-objcopy": ["bitcode-strip", "install-name-tool", "strip"], + "llvm-objdump": ["otool"], + "llvm-rc": ["windres"], + "llvm-readobj": ["readelf"], + "llvm-symbolizer": ["addr2line"], +} + +def _validated_string_list_flag_impl(ctx): + invalid_values = [v for v in ctx.build_setting_value if v not in ctx.attr.values] + if invalid_values: + fail("Tool(s) [{}] are not in the known list of tools: {}".format( + ", ".join(invalid_values), + ", ".join(ctx.attr.values), + )) + return BuildSettingInfo(value = ctx.build_setting_value) + +# Like string_list_flag, but with the validation that string_flag provides. +_validated_string_list_flag = rule( + implementation = _validated_string_list_flag_impl, + build_setting = config.string_list(flag = True), + attrs = { + "values": attr.string_list( + doc = "The list of allowed values for this setting. An error is raised if any other value is given.", + ), + }, + doc = "A string list-typed build setting that can be set on the command line", +) + +def generate_driver_selects(name): + """Generates flags and config settings to configure the tool list. + + By default, all supported tools are included in the "llvm" driver binary. + To build only a subset, specify just the subset you want as the flag. + For example, to produce a binary with just llvm-nm and llvm-size, run: + + $ bazel build \ + --@llvm-project//llvm:driver-tools=llvm-nm,llvm-size \ + @llvm-project//llvm:llvm + + Note: this assumes the flag name is "driver-tools" by being invoked as: + generate_driver_selects(name = "driver-tools") + + Args: + name: the name of the flag that configures which tools are included. + """ + + _validated_string_list_flag( + name = name, + build_setting_default = _TOOLS.keys(), + values = _TOOLS.keys(), + ) + for tool in _TOOLS.keys(): + native.config_setting( + name = "{}-include-{}".format(name, tool), + flag_values = {name: tool}, + ) + +def select_driver_tools(flag): + """Produce a list of tool deps based on generate_driver_selects(). + + Args: + flag: name that was used for generate_driver_selects(). + Returns: + List of tool deps based on generate_driver_selects(). + """ + tools = [] + for tool, target in _TOOLS.items(): + tools += select({ + "{}-include-{}".format(flag, tool): [target], + "//conditions:default": [], + }) + return tools + +def _generate_driver_tools_def_impl(ctx): + # Depending on how the LLVM build files are included, + # it may or may not have the @llvm-project repo prefix. + # Compare just on the name. We could also include the package, + # but the name itself is unique in practice. + label_to_name = {Label(v).name: k for k, v in _TOOLS.items()} + + # Reverse sort by the *main* tool name, but keep aliases together. + # This is consistent with how tools/llvm-driver/CMakeLists.txt does it, + # and this makes sure that more specific tools are checked first. + # For example, "clang-scan-deps" should not match "clang". + tools = [label_to_name[tool.label.name] for tool in ctx.attr.driver_tools] + tool_alias_pairs = [] + for tool_name in reversed(tools): + tool_alias_pairs.append((tool_name, tool_name)) + for extra_alias in _EXTRA_ALIASES.get(tool_name, []): + tool_alias_pairs.append((tool_name, extra_alias)) + + lines = [ + 'LLVM_DRIVER_TOOL("{alias}", {tool})'.format( + tool = tool_name.replace("-", "_"), + alias = alias.removeprefix("llvm-"), + ) + for (tool_name, alias) in tool_alias_pairs + ] + lines.append("#undef LLVM_DRIVER_TOOL") + + ctx.actions.write( + output = ctx.outputs.out, + content = "\n".join(lines), + ) + +generate_driver_tools_def = rule( + implementation = _generate_driver_tools_def_impl, + doc = """Generate a list of LLVM_DRIVER_TOOL macros. +See tools/llvm-driver/CMakeLists.txt for the reference implementation.""", + attrs = { + "driver_tools": attr.label_list( + doc = "List of tools to include in the generated header. Use select_driver_tools() to provide this.", + providers = [CcInfo], + ), + "out": attr.output( + doc = "Name of the generated .def output file.", + mandatory = True, + ), + }, +) + +def llvm_driver_cc_binary( + name, + deps = None, + **kwargs): + """cc_binary wrapper for binaries using the llvm-driver template.""" + expand_template( + name = "_gen_" + name, + out = name + "-driver.cpp", + substitutions = {"@TOOL_NAME@": name.replace("-", "_")}, + template = "//llvm:cmake/modules/llvm-driver-template.cpp.in", + ) + deps = deps or [] + native.cc_binary( + name = name, + srcs = [name + "-driver.cpp"], + deps = deps + ["//llvm:Support"], + **kwargs + ) -- GitLab From 2f05b8905860760f1d4f099dcea0e0dd4b1f49d8 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 29 Mar 2024 09:14:07 -0700 Subject: [PATCH 0006/1690] [RISCV] Move VPseudoBinaryNoMask multiclass to RISCVInstrInfoZvk.td and rename it. NFC Rename to VPseudoBinaryNoMaskTU_Zvk. This more consistent with the naming of the class it instantiates and the _Zvk suffix is used elsewhere in RISCVInstrInfoZvk.td. --- llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td | 13 ------------- llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index 567f4d794608..8cdaa7f2e5ea 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -2133,19 +2133,6 @@ multiclass VPseudoBinary { - let VLMul = MInfo.value, SEW=sew in { - defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX); - def suffix : VPseudoBinaryNoMaskTU; - } -} - multiclass VPseudoBinaryRoundingMode(PseudoToVInst.VInst); } +multiclass VPseudoBinaryNoMaskTU_Zvk { + let VLMul = MInfo.value, SEW=sew in { + defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX); + def suffix : VPseudoBinaryNoMaskTU; + } +} + multiclass VPseudoTernaryNoMask_Zvk("WriteVIALUV_" # mx); defvar ReadVIALUV_MX = !cast("ReadVIALUV_" # mx); - defm _VI : VPseudoBinaryNoMask, + defm _VI : VPseudoBinaryNoMaskTU_Zvk, Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>; } } @@ -317,7 +330,7 @@ multiclass VPseudoVALU_VV_NoMaskTU_Zvk { defvar WriteVIALUV_MX = !cast("WriteVIALUV_" # mx); defvar ReadVIALUV_MX = !cast("ReadVIALUV_" # mx); - defm _VV : VPseudoBinaryNoMask, + defm _VV : VPseudoBinaryNoMaskTU_Zvk, Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>; } } -- GitLab From 661bb9daae8730868d87b7a3cee6b9ad51e988af Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Fri, 29 Mar 2024 12:41:47 -0400 Subject: [PATCH 0007/1690] [GlobalISel] Handle div-by-pow2 (#83155) This patch adds similar handling of div-by-pow2 as in `SelectionDAG`. --- .../llvm/CodeGen/GlobalISel/CombinerHelper.h | 8 + .../include/llvm/Target/GlobalISel/Combine.td | 16 +- .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 91 + .../AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll | 41 +- .../CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll | 147 +- .../CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll | 644 +---- llvm/test/CodeGen/AMDGPU/div_i128.ll | 2298 ++++++++++++++++- 7 files changed, 2504 insertions(+), 741 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 9e8fc5d635c5..28d9cf6260d6 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -673,6 +673,14 @@ public: bool matchSDivByConst(MachineInstr &MI); void applySDivByConst(MachineInstr &MI); + /// Given an G_SDIV \p MI expressing a signed divided by a pow2 constant, + /// return expressions that implements it by shifting. + bool matchDivByPow2(MachineInstr &MI, bool IsSigned); + void applySDivByPow2(MachineInstr &MI); + /// Given an G_UDIV \p MI expressing an unsigned divided by a pow2 constant, + /// return expressions that implements it by shifting. + void applyUDivByPow2(MachineInstr &MI); + // G_UMULH x, (1 << c)) -> x >> (bitwidth - c) bool matchUMulHToLShr(MachineInstr &MI); void applyUMulHToLShr(MachineInstr &MI); diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index 6980cbd04aeb..72d3c0ea69bc 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -179,6 +179,7 @@ def FmArcp : MIFlagEnum<"FmArcp">; def FmContract : MIFlagEnum<"FmContract">; def FmAfn : MIFlagEnum<"FmAfn">; def FmReassoc : MIFlagEnum<"FmReassoc">; +def IsExact : MIFlagEnum<"IsExact">; def MIFlags; // def not; -> Already defined as a SDNode @@ -1036,7 +1037,20 @@ def sdiv_by_const : GICombineRule< [{ return Helper.matchSDivByConst(*${root}); }]), (apply [{ Helper.applySDivByConst(*${root}); }])>; -def intdiv_combines : GICombineGroup<[udiv_by_const, sdiv_by_const]>; +def sdiv_by_pow2 : GICombineRule< + (defs root:$root), + (match (G_SDIV $dst, $x, $y, (MIFlags (not IsExact))):$root, + [{ return Helper.matchDivByPow2(*${root}, /*IsSigned=*/true); }]), + (apply [{ Helper.applySDivByPow2(*${root}); }])>; + +def udiv_by_pow2 : GICombineRule< + (defs root:$root), + (match (G_UDIV $dst, $x, $y, (MIFlags (not IsExact))):$root, + [{ return Helper.matchDivByPow2(*${root}, /*IsSigned=*/false); }]), + (apply [{ Helper.applyUDivByPow2(*${root}); }])>; + +def intdiv_combines : GICombineGroup<[udiv_by_const, sdiv_by_const, + sdiv_by_pow2, udiv_by_pow2]>; def reassoc_ptradd : GICombineRule< (defs root:$root, build_fn_matchinfo:$matchinfo), diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index 2a521b6b068a..f3a9ade51559 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -5270,6 +5270,97 @@ MachineInstr *CombinerHelper::buildSDivUsingMul(MachineInstr &MI) { return MIB.buildMul(Ty, Res, Factor); } +bool CombinerHelper::matchDivByPow2(MachineInstr &MI, bool IsSigned) { + assert((MI.getOpcode() == TargetOpcode::G_SDIV || + MI.getOpcode() == TargetOpcode::G_UDIV) && + "Expected SDIV or UDIV"); + auto &Div = cast(MI); + Register RHS = Div.getReg(2); + auto MatchPow2 = [&](const Constant *C) { + auto *CI = dyn_cast(C); + return CI && (CI->getValue().isPowerOf2() || + (IsSigned && CI->getValue().isNegatedPowerOf2())); + }; + return matchUnaryPredicate(MRI, RHS, MatchPow2, /*AllowUndefs=*/false); +} + +void CombinerHelper::applySDivByPow2(MachineInstr &MI) { + assert(MI.getOpcode() == TargetOpcode::G_SDIV && "Expected SDIV"); + auto &SDiv = cast(MI); + Register Dst = SDiv.getReg(0); + Register LHS = SDiv.getReg(1); + Register RHS = SDiv.getReg(2); + LLT Ty = MRI.getType(Dst); + LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(Ty); + LLT CCVT = + Ty.isVector() ? LLT::vector(Ty.getElementCount(), 1) : LLT::scalar(1); + + Builder.setInstrAndDebugLoc(MI); + + // Effectively we want to lower G_SDIV %lhs, %rhs, where %rhs is a power of 2, + // to the following version: + // + // %c1 = G_CTTZ %rhs + // %inexact = G_SUB $bitwidth, %c1 + // %sign = %G_ASHR %lhs, $(bitwidth - 1) + // %lshr = G_LSHR %sign, %inexact + // %add = G_ADD %lhs, %lshr + // %ashr = G_ASHR %add, %c1 + // %ashr = G_SELECT, %isoneorallones, %lhs, %ashr + // %zero = G_CONSTANT $0 + // %neg = G_NEG %ashr + // %isneg = G_ICMP SLT %rhs, %zero + // %res = G_SELECT %isneg, %neg, %ashr + + unsigned BitWidth = Ty.getScalarSizeInBits(); + auto Zero = Builder.buildConstant(Ty, 0); + + auto Bits = Builder.buildConstant(ShiftAmtTy, BitWidth); + auto C1 = Builder.buildCTTZ(ShiftAmtTy, RHS); + auto Inexact = Builder.buildSub(ShiftAmtTy, Bits, C1); + // Splat the sign bit into the register + auto Sign = Builder.buildAShr( + Ty, LHS, Builder.buildConstant(ShiftAmtTy, BitWidth - 1)); + + // Add (LHS < 0) ? abs2 - 1 : 0; + auto LSrl = Builder.buildLShr(Ty, Sign, Inexact); + auto Add = Builder.buildAdd(Ty, LHS, LSrl); + auto AShr = Builder.buildAShr(Ty, Add, C1); + + // Special case: (sdiv X, 1) -> X + // Special Case: (sdiv X, -1) -> 0-X + auto One = Builder.buildConstant(Ty, 1); + auto MinusOne = Builder.buildConstant(Ty, -1); + auto IsOne = Builder.buildICmp(CmpInst::Predicate::ICMP_EQ, CCVT, RHS, One); + auto IsMinusOne = + Builder.buildICmp(CmpInst::Predicate::ICMP_EQ, CCVT, RHS, MinusOne); + auto IsOneOrMinusOne = Builder.buildOr(CCVT, IsOne, IsMinusOne); + AShr = Builder.buildSelect(Ty, IsOneOrMinusOne, LHS, AShr); + + // If divided by a positive value, we're done. Otherwise, the result must be + // negated. + auto Neg = Builder.buildNeg(Ty, AShr); + auto IsNeg = Builder.buildICmp(CmpInst::Predicate::ICMP_SLT, CCVT, RHS, Zero); + Builder.buildSelect(MI.getOperand(0).getReg(), IsNeg, Neg, AShr); + MI.eraseFromParent(); +} + +void CombinerHelper::applyUDivByPow2(MachineInstr &MI) { + assert(MI.getOpcode() == TargetOpcode::G_UDIV && "Expected UDIV"); + auto &UDiv = cast(MI); + Register Dst = UDiv.getReg(0); + Register LHS = UDiv.getReg(1); + Register RHS = UDiv.getReg(2); + LLT Ty = MRI.getType(Dst); + LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(Ty); + + Builder.setInstrAndDebugLoc(MI); + + auto C1 = Builder.buildCTTZ(ShiftAmtTy, RHS); + Builder.buildLShr(MI.getOperand(0).getReg(), LHS, C1); + MI.eraseFromParent(); +} + bool CombinerHelper::matchUMulHToLShr(MachineInstr &MI) { assert(MI.getOpcode() == TargetOpcode::G_UMULH); Register RHS = MI.getOperand(2).getReg(); diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll index 6eed92ba1d71..6d4aa3b04d76 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll @@ -670,36 +670,19 @@ define amdgpu_kernel void @bfe_sext_in_reg_i24(ptr addrspace(1) %out, ptr addrsp define amdgpu_kernel void @simplify_demanded_bfe_sdiv(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 { ; GFX6-LABEL: simplify_demanded_bfe_sdiv: ; GFX6: ; %bb.0: -; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, 2.0 -; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 -; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 -; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_load_dword s0, s[6:7], 0x0 -; GFX6-NEXT: s_mov_b32 s6, -1 -; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: v_mul_lo_u32 v1, v0, -2 -; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_bfe_i32 s0, s0, 0x100001 -; GFX6-NEXT: s_ashr_i32 s2, s0, 31 -; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX6-NEXT: s_add_i32 s0, s0, s2 -; GFX6-NEXT: s_xor_b32 s0, s0, s2 -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_mul_hi_u32 v0, s0, v0 -; GFX6-NEXT: v_lshlrev_b32_e32 v1, 1, v0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v0 -; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s0, v1 -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 2, v1 -; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX6-NEXT: v_subrev_i32_e64 v2, s[0:1], 2, v1 -; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v0 -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 2, v1 -; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX6-NEXT: v_xor_b32_e32 v0, s2, v0 -; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s2, v0 -; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; GFX6-NEXT: s_load_dword s3, s[2:3], 0x0 +; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: s_bfe_i32 s3, s3, 0x100001 +; GFX6-NEXT: s_ashr_i32 s4, s3, 31 +; GFX6-NEXT: s_lshr_b32 s4, s4, 31 +; GFX6-NEXT: s_add_i32 s3, s3, s4 +; GFX6-NEXT: s_ashr_i32 s3, s3, 1 +; GFX6-NEXT: v_mov_b32_e32 v0, s3 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX6-NEXT: s_endpgm %src = load i32, ptr addrspace(1) %in, align 4 %bfe = call i32 @llvm.amdgcn.sbfe.i32(i32 %src, i32 1, i32 16) diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll index 1061f0003bd4..2c2f8e914447 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll @@ -279,125 +279,27 @@ define i32 @v_sdiv_i32_pow2k_denom(i32 %num) { ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CHECK-NEXT: v_ashrrev_i32_e32 v1, 31, v0 -; CHECK-NEXT: v_rcp_iflag_f32_e32 v2, 0x45800000 -; CHECK-NEXT: v_mov_b32_e32 v3, 0xfffff000 -; CHECK-NEXT: v_mov_b32_e32 v4, 0x1000 +; CHECK-NEXT: v_lshrrev_b32_e32 v1, 20, v1 ; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; CHECK-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 -; CHECK-NEXT: v_xor_b32_e32 v0, v0, v1 -; CHECK-NEXT: v_cvt_u32_f32_e32 v2, v2 -; CHECK-NEXT: v_mul_lo_u32 v3, v2, v3 -; CHECK-NEXT: v_mul_hi_u32 v3, v2, v3 -; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; CHECK-NEXT: v_mul_hi_u32 v2, v0, v2 -; CHECK-NEXT: v_lshlrev_b32_e32 v3, 12, v2 -; CHECK-NEXT: v_add_i32_e32 v5, vcc, 1, v2 -; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v3 -; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v4 -; CHECK-NEXT: v_cndmask_b32_e64 v2, v2, v5, s[4:5] -; CHECK-NEXT: v_subrev_i32_e32 v3, vcc, 0x1000, v0 -; CHECK-NEXT: v_cndmask_b32_e64 v0, v0, v3, s[4:5] -; CHECK-NEXT: v_add_i32_e32 v3, vcc, 1, v2 -; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc -; CHECK-NEXT: v_xor_b32_e32 v0, v0, v1 -; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 +; CHECK-NEXT: v_ashrrev_i32_e32 v0, 12, v0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %result = sdiv i32 %num, 4096 ret i32 %result } define <2 x i32> @v_sdiv_v2i32_pow2k_denom(<2 x i32> %num) { -; GISEL-LABEL: v_sdiv_v2i32_pow2k_denom: -; GISEL: ; %bb.0: -; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GISEL-NEXT: v_ashrrev_i32_e32 v2, 31, v0 -; GISEL-NEXT: v_mov_b32_e32 v3, 0x1000 -; GISEL-NEXT: v_cvt_f32_u32_e32 v4, 0x1000 -; GISEL-NEXT: v_mov_b32_e32 v5, 0xfffff000 -; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v1 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GISEL-NEXT: v_rcp_iflag_f32_e32 v4, v4 -; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v6 -; GISEL-NEXT: v_xor_b32_e32 v0, v0, v2 -; GISEL-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 -; GISEL-NEXT: v_xor_b32_e32 v1, v1, v6 -; GISEL-NEXT: v_cvt_u32_f32_e32 v4, v4 -; GISEL-NEXT: v_mul_lo_u32 v5, v4, v5 -; GISEL-NEXT: v_mul_hi_u32 v5, v4, v5 -; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GISEL-NEXT: v_mul_hi_u32 v5, v0, v4 -; GISEL-NEXT: v_mul_hi_u32 v4, v1, v4 -; GISEL-NEXT: v_lshlrev_b32_e32 v7, 12, v5 -; GISEL-NEXT: v_add_i32_e32 v8, vcc, 1, v5 -; GISEL-NEXT: v_lshlrev_b32_e32 v9, 12, v4 -; GISEL-NEXT: v_add_i32_e32 v10, vcc, 1, v4 -; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v7 -; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v9 -; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v3 -; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[4:5] -; GISEL-NEXT: v_sub_i32_e32 v7, vcc, v0, v3 -; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v1, v3 -; GISEL-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[6:7] -; GISEL-NEXT: v_subrev_i32_e32 v8, vcc, 0x1000, v1 -; GISEL-NEXT: v_cndmask_b32_e64 v0, v0, v7, s[4:5] -; GISEL-NEXT: v_add_i32_e32 v7, vcc, 1, v5 -; GISEL-NEXT: v_cndmask_b32_e64 v1, v1, v8, s[6:7] -; GISEL-NEXT: v_add_i32_e32 v8, vcc, 1, v4 -; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v3 -; GISEL-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc -; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 -; GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v8, vcc -; GISEL-NEXT: v_xor_b32_e32 v0, v0, v2 -; GISEL-NEXT: v_xor_b32_e32 v1, v1, v6 -; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 -; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v6 -; GISEL-NEXT: s_setpc_b64 s[30:31] -; -; CGP-LABEL: v_sdiv_v2i32_pow2k_denom: -; CGP: ; %bb.0: -; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CGP-NEXT: v_ashrrev_i32_e32 v2, 31, v0 -; CGP-NEXT: v_rcp_iflag_f32_e32 v3, 0x45800000 -; CGP-NEXT: v_mov_b32_e32 v4, 0xfffff000 -; CGP-NEXT: v_mov_b32_e32 v5, 0x1000 -; CGP-NEXT: v_ashrrev_i32_e32 v6, 31, v1 -; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; CGP-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3 -; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v6 -; CGP-NEXT: v_xor_b32_e32 v0, v0, v2 -; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3 -; CGP-NEXT: v_xor_b32_e32 v1, v1, v6 -; CGP-NEXT: v_mul_lo_u32 v4, v3, v4 -; CGP-NEXT: v_mul_hi_u32 v4, v3, v4 -; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4 -; CGP-NEXT: v_mul_hi_u32 v4, v0, v3 -; CGP-NEXT: v_mul_hi_u32 v3, v1, v3 -; CGP-NEXT: v_lshlrev_b32_e32 v7, 12, v4 -; CGP-NEXT: v_add_i32_e32 v8, vcc, 1, v4 -; CGP-NEXT: v_lshlrev_b32_e32 v9, 12, v3 -; CGP-NEXT: v_add_i32_e32 v10, vcc, 1, v3 -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v7 -; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v9 -; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v5 -; CGP-NEXT: v_cndmask_b32_e64 v4, v4, v8, s[4:5] -; CGP-NEXT: v_sub_i32_e32 v7, vcc, v0, v5 -; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v1, v5 -; CGP-NEXT: v_cndmask_b32_e64 v3, v3, v10, s[6:7] -; CGP-NEXT: v_subrev_i32_e32 v8, vcc, 0x1000, v1 -; CGP-NEXT: v_cndmask_b32_e64 v0, v0, v7, s[4:5] -; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v4 -; CGP-NEXT: v_cndmask_b32_e64 v1, v1, v8, s[6:7] -; CGP-NEXT: v_add_i32_e32 v8, vcc, 1, v3 -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5 -; CGP-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5 -; CGP-NEXT: v_cndmask_b32_e32 v1, v3, v8, vcc -; CGP-NEXT: v_xor_b32_e32 v0, v0, v2 -; CGP-NEXT: v_xor_b32_e32 v1, v1, v6 -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 -; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v6 -; CGP-NEXT: s_setpc_b64 s[30:31] +; CHECK-LABEL: v_sdiv_v2i32_pow2k_denom: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_ashrrev_i32_e32 v2, 31, v0 +; CHECK-NEXT: v_ashrrev_i32_e32 v3, 31, v1 +; CHECK-NEXT: v_lshrrev_b32_e32 v2, 20, v2 +; CHECK-NEXT: v_lshrrev_b32_e32 v3, 20, v3 +; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; CHECK-NEXT: v_ashrrev_i32_e32 v0, 12, v0 +; CHECK-NEXT: v_ashrrev_i32_e32 v1, 12, v1 +; CHECK-NEXT: s_setpc_b64 s[30:31] %result = sdiv <2 x i32> %num, ret <2 x i32> %result } @@ -884,3 +786,24 @@ define <2 x i32> @v_sdiv_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) { %result = sdiv <2 x i32> %num.mask, %den.mask ret <2 x i32> %result } + +define i32 @v_sdiv_i32_exact(i32 %num) { +; CHECK-LABEL: v_sdiv_i32_exact: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_ashrrev_i32_e32 v0, 12, v0 +; CHECK-NEXT: s_setpc_b64 s[30:31] + %result = sdiv exact i32 %num, 4096 + ret i32 %result +} + +define <2 x i32> @v_sdiv_v2i32_exact(<2 x i32> %num) { +; CHECK-LABEL: v_sdiv_v2i32_exact: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_ashrrev_i32_e32 v0, 12, v0 +; CHECK-NEXT: v_ashrrev_i32_e32 v1, 10, v1 +; CHECK-NEXT: s_setpc_b64 s[30:31] + %result = sdiv exact <2 x i32> %num, + ret <2 x i32> %result +} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll index 84906c01a469..377fa24cb475 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll @@ -999,602 +999,45 @@ define i64 @v_sdiv_i64_pow2k_denom(i64 %num) { ; CHECK-LABEL: v_sdiv_i64_pow2k_denom: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_cvt_f32_u32_e32 v2, 0x1000 -; CHECK-NEXT: v_cvt_f32_ubyte0_e32 v3, 0 -; CHECK-NEXT: v_mov_b32_e32 v6, 0xfffff000 -; CHECK-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3 -; CHECK-NEXT: v_rcp_iflag_f32_e32 v2, v2 -; CHECK-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 -; CHECK-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 -; CHECK-NEXT: v_trunc_f32_e32 v4, v3 -; CHECK-NEXT: v_mac_f32_e32 v2, 0xcf800000, v4 -; CHECK-NEXT: v_cvt_u32_f32_e32 v5, v2 -; CHECK-NEXT: v_cvt_u32_f32_e32 v7, v4 -; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v5, 0 -; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v7, v[3:4] -; CHECK-NEXT: v_mul_hi_u32 v8, v5, v2 -; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], -1, v5, v[3:4] -; CHECK-NEXT: v_mul_lo_u32 v4, v7, v2 -; CHECK-NEXT: v_mul_hi_u32 v2, v7, v2 -; CHECK-NEXT: v_mul_lo_u32 v9, v5, v3 -; CHECK-NEXT: v_mul_lo_u32 v10, v7, v3 -; CHECK-NEXT: v_mul_hi_u32 v11, v5, v3 -; CHECK-NEXT: v_mul_hi_u32 v3, v7, v3 -; CHECK-NEXT: v_add_i32_e32 v4, vcc, v4, v9 -; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v2, vcc, v10, v2 -; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v4, vcc, v4, v8 -; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v4, vcc, v9, v4 -; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v11 -; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v8, vcc, v10, v8 -; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v4, vcc, v8, v4 -; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4 -; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v2 -; CHECK-NEXT: v_addc_u32_e32 v7, vcc, v7, v3, vcc -; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v5, 0 -; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v7, v[3:4] -; CHECK-NEXT: v_ashrrev_i32_e32 v6, 31, v1 -; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v6 -; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], -1, v5, v[3:4] -; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; CHECK-NEXT: v_xor_b32_e32 v4, v0, v6 -; CHECK-NEXT: v_mul_lo_u32 v0, v7, v2 -; CHECK-NEXT: v_mul_lo_u32 v8, v5, v3 -; CHECK-NEXT: v_xor_b32_e32 v9, v1, v6 -; CHECK-NEXT: v_mul_hi_u32 v1, v5, v2 -; CHECK-NEXT: v_mul_hi_u32 v2, v7, v2 -; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v8 -; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc -; CHECK-NEXT: v_mul_lo_u32 v1, v7, v3 -; CHECK-NEXT: v_add_i32_e32 v0, vcc, v8, v0 -; CHECK-NEXT: v_mul_hi_u32 v8, v5, v3 -; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2 -; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v8 -; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v8 -; CHECK-NEXT: v_mul_hi_u32 v3, v7, v3 -; CHECK-NEXT: v_add_i32_e32 v0, vcc, v1, v0 -; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v1, vcc, v2, v1 -; CHECK-NEXT: v_add_i32_e32 v1, vcc, v3, v1 -; CHECK-NEXT: v_add_i32_e32 v0, vcc, v5, v0 -; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc -; CHECK-NEXT: v_mul_lo_u32 v2, v9, v0 -; CHECK-NEXT: v_mul_lo_u32 v3, v4, v1 -; CHECK-NEXT: v_mul_hi_u32 v7, v4, v0 -; CHECK-NEXT: v_mul_hi_u32 v0, v9, v0 -; CHECK-NEXT: v_mov_b32_e32 v5, 0x1000 -; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v7 -; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; CHECK-NEXT: v_mul_lo_u32 v7, v9, v1 -; CHECK-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; CHECK-NEXT: v_mul_hi_u32 v3, v4, v1 -; CHECK-NEXT: v_add_i32_e32 v0, vcc, v7, v0 -; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v3, vcc, v7, v3 -; CHECK-NEXT: v_add_i32_e32 v7, vcc, v0, v2 -; CHECK-NEXT: v_mul_hi_u32 v8, v9, v1 -; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v7, 0 -; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; CHECK-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; CHECK-NEXT: v_add_i32_e32 v3, vcc, v8, v2 -; CHECK-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v5, v3, v[1:2] -; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v4, v0 -; CHECK-NEXT: v_subb_u32_e64 v2, s[4:5], v9, v1, vcc -; CHECK-NEXT: v_sub_i32_e64 v1, s[4:5], v9, v1 -; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc -; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v5 -; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v5 -; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[4:5] -; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2 -; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc -; CHECK-NEXT: v_cndmask_b32_e64 v2, -1, v4, s[4:5] -; CHECK-NEXT: v_add_i32_e32 v4, vcc, 1, v7 -; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v3, vcc -; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5 -; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc -; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; CHECK-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc -; CHECK-NEXT: v_add_i32_e32 v1, vcc, 1, v4 -; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v8, vcc -; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc -; CHECK-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc -; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v7, v0, vcc -; CHECK-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; CHECK-NEXT: v_xor_b32_e32 v0, v0, v6 -; CHECK-NEXT: v_xor_b32_e32 v1, v1, v6 -; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 -; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v6, vcc -; CHECK-NEXT: s_setpc_b64 s[30:31] - %result = sdiv i64 %num, 4096 - ret i64 %result -} - -define <2 x i64> @v_sdiv_v2i64_pow2k_denom(<2 x i64> %num) { -; GISEL-LABEL: v_sdiv_v2i64_pow2k_denom: -; GISEL: ; %bb.0: -; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GISEL-NEXT: v_cvt_f32_u32_e32 v4, 0x1000 -; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v5, 0 -; GISEL-NEXT: s_sub_u32 s6, 0, 0x1000 -; GISEL-NEXT: s_subb_u32 s7, 0, 0 -; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5 -; GISEL-NEXT: v_rcp_iflag_f32_e32 v4, v4 -; GISEL-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4 -; GISEL-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4 -; GISEL-NEXT: v_trunc_f32_e32 v7, v5 -; GISEL-NEXT: v_mac_f32_e32 v4, 0xcf800000, v7 -; GISEL-NEXT: v_cvt_u32_f32_e32 v6, v4 -; GISEL-NEXT: v_cvt_u32_f32_e32 v7, v7 -; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s6, v6, 0 -; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], s6, v7, v[5:6] -; GISEL-NEXT: v_mul_lo_u32 v5, v7, v4 -; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], s7, v6, v[8:9] -; GISEL-NEXT: v_mul_hi_u32 v9, v6, v4 -; GISEL-NEXT: v_mul_hi_u32 v4, v7, v4 -; GISEL-NEXT: v_mul_lo_u32 v10, v6, v8 -; GISEL-NEXT: v_mul_lo_u32 v11, v7, v8 -; GISEL-NEXT: v_mul_hi_u32 v12, v6, v8 -; GISEL-NEXT: v_mul_hi_u32 v8, v7, v8 -; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v10 -; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v9 -; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v5, vcc, v10, v5 -; GISEL-NEXT: v_add_i32_e32 v4, vcc, v11, v4 -; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v12 -; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10 -; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v5, vcc, v9, v5 -; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5 -; GISEL-NEXT: v_add_i32_e32 v11, vcc, v6, v4 -; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], s6, v11, 0 -; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v7, v5, vcc -; GISEL-NEXT: v_mov_b32_e32 v4, v9 -; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], s6, v5, v[4:5] -; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], s7, v11, v[9:10] -; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc -; GISEL-NEXT: v_xor_b32_e32 v10, v0, v4 -; GISEL-NEXT: v_mul_lo_u32 v0, v5, v8 -; GISEL-NEXT: v_mul_lo_u32 v12, v11, v9 -; GISEL-NEXT: v_xor_b32_e32 v13, v1, v4 -; GISEL-NEXT: v_mul_hi_u32 v1, v11, v8 -; GISEL-NEXT: v_mul_hi_u32 v8, v5, v8 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12 -; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc -; GISEL-NEXT: v_mul_lo_u32 v1, v5, v9 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0 -; GISEL-NEXT: v_mul_hi_u32 v12, v11, v9 -; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v8 -; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v12 -; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v12 -; GISEL-NEXT: v_mul_hi_u32 v9, v5, v9 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v1, v0 -; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v1, vcc, v8, v1 -; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0 -; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v5, v1, vcc -; GISEL-NEXT: v_mul_lo_u32 v8, v13, v0 -; GISEL-NEXT: v_mul_lo_u32 v9, v10, v1 -; GISEL-NEXT: v_mul_hi_u32 v11, v10, v0 -; GISEL-NEXT: v_mul_hi_u32 v0, v13, v0 -; GISEL-NEXT: v_mov_b32_e32 v5, 0x1000 -; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9 -; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v11 -; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc -; GISEL-NEXT: v_mul_lo_u32 v11, v13, v1 -; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GISEL-NEXT: v_mul_hi_u32 v9, v10, v1 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0 -; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9 -; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v9 -; GISEL-NEXT: v_add_i32_e32 v11, vcc, v0, v8 -; GISEL-NEXT: v_mul_hi_u32 v12, v13, v1 -; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v11, 0 -; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v8 -; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v12, v[1:2] -; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v10, v0 -; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], 0, v11, v[8:9] -; GISEL-NEXT: s_sub_u32 s6, 0, 0x1000 -; GISEL-NEXT: s_subb_u32 s7, 0, 0 -; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], v13, v8, vcc -; GISEL-NEXT: v_sub_i32_e64 v8, s[4:5], v13, v8 -; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v5 -; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5] -; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1 -; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v8, vcc -; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v0, v5 -; GISEL-NEXT: v_cndmask_b32_e64 v10, -1, v9, s[4:5] -; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v1, vcc -; GISEL-NEXT: v_add_i32_e32 v13, vcc, 1, v11 -; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v6, 0 -; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v12, vcc -; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v8, v5 -; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc -; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 -; GISEL-NEXT: v_cndmask_b32_e32 v15, -1, v8, vcc -; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], s6, v7, v[1:2] -; GISEL-NEXT: v_add_i32_e32 v1, vcc, 1, v13 -; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], s7, v6, v[8:9] -; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v14, vcc -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v15 -; GISEL-NEXT: v_cndmask_b32_e32 v9, v13, v1, vcc -; GISEL-NEXT: v_mul_lo_u32 v1, v7, v0 -; GISEL-NEXT: v_mul_lo_u32 v13, v6, v8 -; GISEL-NEXT: v_mul_hi_u32 v15, v6, v0 -; GISEL-NEXT: v_cndmask_b32_e32 v14, v14, v16, vcc -; GISEL-NEXT: v_mul_hi_u32 v0, v7, v0 -; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v13 -; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v15 -; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc -; GISEL-NEXT: v_mul_lo_u32 v15, v7, v8 -; GISEL-NEXT: v_add_i32_e32 v1, vcc, v13, v1 -; GISEL-NEXT: v_mul_hi_u32 v13, v6, v8 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v15, v0 -; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v13 -; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v13, vcc, v15, v13 -; GISEL-NEXT: v_mul_hi_u32 v8, v7, v8 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v1, vcc, v13, v1 -; GISEL-NEXT: v_add_i32_e32 v1, vcc, v8, v1 -; GISEL-NEXT: v_add_i32_e32 v8, vcc, v6, v0 -; GISEL-NEXT: v_addc_u32_e32 v13, vcc, v7, v1, vcc -; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0 -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10 -; GISEL-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc -; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], s6, v13, v[1:2] -; GISEL-NEXT: v_xor_b32_e32 v1, v9, v4 -; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v3 -; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], s7, v8, v[6:7] -; GISEL-NEXT: v_cndmask_b32_e32 v10, v12, v14, vcc -; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v9 -; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v9, vcc -; GISEL-NEXT: v_xor_b32_e32 v11, v2, v9 -; GISEL-NEXT: v_mul_lo_u32 v2, v13, v0 -; GISEL-NEXT: v_mul_lo_u32 v7, v8, v6 -; GISEL-NEXT: v_xor_b32_e32 v12, v3, v9 -; GISEL-NEXT: v_mul_hi_u32 v3, v8, v0 -; GISEL-NEXT: v_mul_hi_u32 v0, v13, v0 -; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v7 -; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GISEL-NEXT: v_mul_lo_u32 v3, v13, v6 -; GISEL-NEXT: v_add_i32_e32 v2, vcc, v7, v2 -; GISEL-NEXT: v_mul_hi_u32 v7, v8, v6 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v3, v0 -; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v7 -; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v7 -; GISEL-NEXT: v_mul_hi_u32 v6, v13, v6 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GISEL-NEXT: v_add_i32_e32 v2, vcc, v6, v2 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0 -; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v13, v2, vcc -; GISEL-NEXT: v_mul_lo_u32 v3, v12, v0 -; GISEL-NEXT: v_mul_lo_u32 v6, v11, v2 -; GISEL-NEXT: v_mul_hi_u32 v7, v11, v0 -; GISEL-NEXT: v_mul_hi_u32 v0, v12, v0 -; GISEL-NEXT: v_xor_b32_e32 v8, v10, v4 -; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v6 -; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v7 -; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc -; GISEL-NEXT: v_mul_lo_u32 v7, v12, v2 -; GISEL-NEXT: v_add_i32_e32 v3, vcc, v6, v3 -; GISEL-NEXT: v_mul_hi_u32 v6, v11, v2 -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v7, v0 -; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v6 -; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GISEL-NEXT: v_add_i32_e32 v10, vcc, v0, v3 -; GISEL-NEXT: v_mul_hi_u32 v7, v12, v2 -; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v10, 0 -; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc -; GISEL-NEXT: v_add_i32_e32 v0, vcc, v6, v0 -; GISEL-NEXT: v_add_i32_e32 v13, vcc, v7, v0 -; GISEL-NEXT: v_mov_b32_e32 v0, v3 -; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v5, v13, v[0:1] -; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4 -; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v8, v4, vcc -; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], 0, v10, v[6:7] -; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v11, v2 -; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v12, v3, vcc -; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v12, v3 -; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc -; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v5 -; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v5 -; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5] -; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v4 -; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc -; GISEL-NEXT: v_cndmask_b32_e64 v4, -1, v6, s[4:5] -; GISEL-NEXT: v_add_i32_e32 v6, vcc, 1, v10 -; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v13, vcc -; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v5 -; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc -; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 -; GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc -; GISEL-NEXT: v_add_i32_e32 v3, vcc, 1, v6 -; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 -; GISEL-NEXT: v_cndmask_b32_e32 v2, v6, v3, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v3, v7, v5, vcc -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 -; GISEL-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v3, v13, v3, vcc -; GISEL-NEXT: v_xor_b32_e32 v2, v2, v9 -; GISEL-NEXT: v_xor_b32_e32 v3, v3, v9 -; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v9 -; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v9, vcc +; CHECK-NEXT: v_ashrrev_i32_e32 v2, 31, v1 +; CHECK-NEXT: v_lshrrev_b32_e32 v2, 20, v2 +; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; CHECK-NEXT: v_ashr_i64 v[0:1], v[0:1], 12 +; CHECK-NEXT: s_setpc_b64 s[30:31] + %result = sdiv i64 %num, 4096 + ret i64 %result +} + +define <2 x i64> @v_sdiv_v2i64_pow2k_denom(<2 x i64> %num) { +; GISEL-LABEL: v_sdiv_v2i64_pow2k_denom: +; GISEL: ; %bb.0: +; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1 +; GISEL-NEXT: v_lshrrev_b32_e32 v4, 20, v4 +; GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v3 +; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4 +; GISEL-NEXT: v_lshrrev_b32_e32 v5, 20, v5 +; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v5 +; GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; GISEL-NEXT: v_ashr_i64 v[0:1], v[0:1], 12 +; GISEL-NEXT: v_ashr_i64 v[2:3], v[2:3], 12 ; GISEL-NEXT: s_setpc_b64 s[30:31] ; ; CGP-LABEL: v_sdiv_v2i64_pow2k_denom: ; CGP: ; %bb.0: ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CGP-NEXT: v_cvt_f32_u32_e32 v4, 0x1000 -; CGP-NEXT: v_cvt_f32_ubyte0_e32 v5, 0 -; CGP-NEXT: v_mov_b32_e32 v6, 0xfffff000 -; CGP-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5 -; CGP-NEXT: v_rcp_iflag_f32_e32 v4, v4 -; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4 -; CGP-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4 -; CGP-NEXT: v_trunc_f32_e32 v7, v5 -; CGP-NEXT: v_mac_f32_e32 v4, 0xcf800000, v7 -; CGP-NEXT: v_cvt_u32_f32_e32 v8, v4 -; CGP-NEXT: v_cvt_u32_f32_e32 v9, v7 -; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v8, 0 -; CGP-NEXT: v_mov_b32_e32 v7, v5 -; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v6, v9, v[7:8] -; CGP-NEXT: v_mul_hi_u32 v12, v9, v4 -; CGP-NEXT: v_mad_u64_u32 v[13:14], s[4:5], -1, v8, v[10:11] -; CGP-NEXT: v_mul_lo_u32 v10, v9, v4 -; CGP-NEXT: v_mul_hi_u32 v11, v8, v4 -; CGP-NEXT: v_mul_lo_u32 v4, v8, v13 -; CGP-NEXT: v_mul_lo_u32 v7, v9, v13 -; CGP-NEXT: v_mul_hi_u32 v14, v8, v13 -; CGP-NEXT: v_mul_hi_u32 v13, v9, v13 -; CGP-NEXT: v_add_i32_e32 v4, vcc, v10, v4 -; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11 -; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v4, vcc, v15, v4 -; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v12 -; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v14 -; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v14, vcc, v15, v14 -; CGP-NEXT: v_add_i32_e32 v4, vcc, v7, v4 -; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v7, vcc, v14, v7 -; CGP-NEXT: v_add_i32_e32 v7, vcc, v13, v7 -; CGP-NEXT: v_add_i32_e32 v16, vcc, v8, v4 -; CGP-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v6, v16, 0 -; CGP-NEXT: v_addc_u32_e32 v17, vcc, v9, v7, vcc -; CGP-NEXT: v_mov_b32_e32 v4, v14 -; CGP-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v6, v17, v[4:5] -; CGP-NEXT: v_ashrrev_i32_e32 v7, 31, v1 -; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v7 -; CGP-NEXT: v_mad_u64_u32 v[14:15], s[4:5], -1, v16, v[14:15] -; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc -; CGP-NEXT: v_xor_b32_e32 v15, v0, v7 -; CGP-NEXT: v_mul_lo_u32 v0, v17, v13 -; CGP-NEXT: v_mul_lo_u32 v4, v16, v14 -; CGP-NEXT: v_xor_b32_e32 v18, v1, v7 -; CGP-NEXT: v_mul_hi_u32 v1, v16, v13 -; CGP-NEXT: v_mul_hi_u32 v13, v17, v13 +; CGP-NEXT: v_ashrrev_i32_e32 v4, 31, v1 +; CGP-NEXT: v_lshrrev_b32_e32 v4, 20, v4 ; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc -; CGP-NEXT: v_mul_lo_u32 v1, v17, v14 -; CGP-NEXT: v_add_i32_e32 v0, vcc, v4, v0 -; CGP-NEXT: v_mul_hi_u32 v4, v16, v14 -; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v13 -; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v4 -; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v4, vcc, v13, v4 -; CGP-NEXT: v_mul_hi_u32 v13, v17, v14 -; CGP-NEXT: v_add_i32_e32 v0, vcc, v1, v0 -; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v1, vcc, v4, v1 -; CGP-NEXT: v_add_i32_e32 v1, vcc, v13, v1 -; CGP-NEXT: v_add_i32_e32 v0, vcc, v16, v0 -; CGP-NEXT: v_addc_u32_e32 v1, vcc, v17, v1, vcc -; CGP-NEXT: v_mul_lo_u32 v13, v18, v0 -; CGP-NEXT: v_mul_lo_u32 v14, v15, v1 -; CGP-NEXT: v_mul_hi_u32 v16, v15, v0 -; CGP-NEXT: v_mul_hi_u32 v0, v18, v0 -; CGP-NEXT: v_mov_b32_e32 v4, 0x1000 -; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v14 -; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v16 -; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc -; CGP-NEXT: v_mul_lo_u32 v16, v18, v1 -; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13 -; CGP-NEXT: v_mul_hi_u32 v14, v15, v1 -; CGP-NEXT: v_add_i32_e32 v0, vcc, v16, v0 -; CGP-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v14 -; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v14, vcc, v16, v14 -; CGP-NEXT: v_add_i32_e32 v16, vcc, v0, v13 -; CGP-NEXT: v_mul_hi_u32 v17, v18, v1 -; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v16, 0 -; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13 -; CGP-NEXT: v_add_i32_e32 v17, vcc, v17, v13 -; CGP-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v4, v17, v[1:2] -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v15, v0 -; CGP-NEXT: v_subb_u32_e64 v1, s[4:5], v18, v13, vcc -; CGP-NEXT: v_sub_i32_e64 v13, s[4:5], v18, v13 -; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v4 -; CGP-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[4:5] -; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1 -; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v13, vcc -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 -; CGP-NEXT: v_subbrev_u32_e32 v13, vcc, 0, v1, vcc -; CGP-NEXT: v_add_i32_e32 v15, vcc, 1, v16 -; CGP-NEXT: v_addc_u32_e32 v18, vcc, 0, v17, vcc -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4 -; CGP-NEXT: v_mov_b32_e32 v0, v5 -; CGP-NEXT: v_cndmask_b32_e64 v14, -1, v14, s[4:5] -; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v9, v[0:1] -; CGP-NEXT: v_cndmask_b32_e64 v19, 0, -1, vcc -; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v13 -; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], -1, v8, v[0:1] -; CGP-NEXT: v_cndmask_b32_e32 v5, -1, v19, vcc -; CGP-NEXT: v_add_i32_e32 v1, vcc, 1, v15 -; CGP-NEXT: v_mul_lo_u32 v19, v8, v0 -; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v18, vcc -; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 -; CGP-NEXT: v_cndmask_b32_e32 v5, v15, v1, vcc -; CGP-NEXT: v_cndmask_b32_e32 v13, v18, v13, vcc -; CGP-NEXT: v_add_i32_e32 v1, vcc, v10, v19 -; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v11 -; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc -; CGP-NEXT: v_mul_lo_u32 v11, v9, v0 -; CGP-NEXT: v_add_i32_e32 v1, vcc, v10, v1 -; CGP-NEXT: v_mul_hi_u32 v10, v8, v0 -; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12 -; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10 -; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11 -; CGP-NEXT: v_mul_hi_u32 v0, v9, v0 -; CGP-NEXT: v_add_i32_e32 v1, vcc, v10, v1 -; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10 -; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v10 -; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v1 -; CGP-NEXT: v_addc_u32_e32 v9, vcc, v9, v0, vcc -; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v8, 0 -; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v14 -; CGP-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; CGP-NEXT: v_xor_b32_e32 v11, v5, v7 -; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v6, v9, v[1:2] -; CGP-NEXT: v_cndmask_b32_e32 v10, v17, v13, vcc -; CGP-NEXT: v_xor_b32_e32 v1, v10, v7 -; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], -1, v8, v[5:6] -; CGP-NEXT: v_ashrrev_i32_e32 v10, 31, v3 -; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v10 -; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v10, vcc -; CGP-NEXT: v_xor_b32_e32 v12, v2, v10 -; CGP-NEXT: v_mul_lo_u32 v2, v9, v0 -; CGP-NEXT: v_mul_lo_u32 v6, v8, v5 -; CGP-NEXT: v_xor_b32_e32 v13, v3, v10 -; CGP-NEXT: v_mul_hi_u32 v3, v8, v0 -; CGP-NEXT: v_mul_hi_u32 v0, v9, v0 -; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v6 -; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; CGP-NEXT: v_mul_lo_u32 v3, v9, v5 -; CGP-NEXT: v_add_i32_e32 v2, vcc, v6, v2 -; CGP-NEXT: v_mul_hi_u32 v6, v8, v5 -; CGP-NEXT: v_add_i32_e32 v0, vcc, v3, v0 -; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v6 -; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v6 -; CGP-NEXT: v_mul_hi_u32 v5, v9, v5 -; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; CGP-NEXT: v_add_i32_e32 v2, vcc, v5, v2 -; CGP-NEXT: v_add_i32_e32 v3, vcc, v8, v0 -; CGP-NEXT: v_addc_u32_e32 v2, vcc, v9, v2, vcc -; CGP-NEXT: v_mul_lo_u32 v5, v13, v3 -; CGP-NEXT: v_mul_lo_u32 v6, v12, v2 -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v11, v7 -; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v7, vcc -; CGP-NEXT: v_mul_hi_u32 v7, v12, v3 -; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v7 -; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc -; CGP-NEXT: v_mul_lo_u32 v7, v13, v2 -; CGP-NEXT: v_mul_hi_u32 v3, v13, v3 -; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; CGP-NEXT: v_mul_hi_u32 v6, v12, v2 -; CGP-NEXT: v_add_i32_e32 v3, vcc, v7, v3 -; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v6 -; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; CGP-NEXT: v_add_i32_e32 v7, vcc, v3, v5 -; CGP-NEXT: v_mul_hi_u32 v8, v13, v2 -; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, v7, 0 -; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc -; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v5 -; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v4, v8, v[3:4] -; CGP-NEXT: v_sub_i32_e32 v2, vcc, v12, v2 -; CGP-NEXT: v_subb_u32_e64 v3, s[4:5], v13, v5, vcc -; CGP-NEXT: v_sub_i32_e64 v5, s[4:5], v13, v5 -; CGP-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc -; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v4 -; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v4 -; CGP-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5] -; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3 -; CGP-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc -; CGP-NEXT: v_cndmask_b32_e64 v3, -1, v6, s[4:5] -; CGP-NEXT: v_add_i32_e32 v6, vcc, 1, v7 -; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v8, vcc -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v4 -; CGP-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc -; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 -; CGP-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc -; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v6 -; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v9, vcc -; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 -; CGP-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc -; CGP-NEXT: v_cndmask_b32_e32 v4, v9, v5, vcc -; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3 -; CGP-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; CGP-NEXT: v_cndmask_b32_e32 v3, v8, v4, vcc -; CGP-NEXT: v_xor_b32_e32 v2, v2, v10 -; CGP-NEXT: v_xor_b32_e32 v3, v3, v10 -; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v10 -; CGP-NEXT: v_subb_u32_e32 v3, vcc, v3, v10, vcc +; CGP-NEXT: v_ashrrev_i32_e32 v4, 31, v3 +; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; CGP-NEXT: v_lshrrev_b32_e32 v4, 20, v4 +; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4 +; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; CGP-NEXT: v_ashr_i64 v[0:1], v[0:1], 12 +; CGP-NEXT: v_ashr_i64 v[2:3], v[2:3], 12 ; CGP-NEXT: s_setpc_b64 s[30:31] %result = sdiv <2 x i64> %num, ret <2 x i64> %result @@ -3398,3 +2841,24 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) { %result = sdiv <2 x i64> %num.mask, %den.mask ret <2 x i64> %result } + +define i64 @v_sdiv_i64_exact(i64 %num) { +; CHECK-LABEL: v_sdiv_i64_exact: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_ashr_i64 v[0:1], v[0:1], 12 +; CHECK-NEXT: s_setpc_b64 s[30:31] + %result = sdiv exact i64 %num, 4096 + ret i64 %result +} + +define <2 x i64> @v_sdiv_v2i64_exact(<2 x i64> %num) { +; CHECK-LABEL: v_sdiv_v2i64_exact: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_ashr_i64 v[0:1], v[0:1], 12 +; CHECK-NEXT: v_ashr_i64 v[2:3], v[2:3], 10 +; CHECK-NEXT: s_setpc_b64 s[30:31] + %result = sdiv exact <2 x i64> %num, + ret <2 x i64> %result +} diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll index 2f3d5d9d140c..cf99b5d80e13 100644 --- a/llvm/test/CodeGen/AMDGPU/div_i128.ll +++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll @@ -1,10 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s -; RUN: llc -O0 -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9-O0,GFX9-SDAG-O0 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9 %s +; RUN: llc -O0 -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9-O0 %s -; FIXME: GlobalISel missing the power-of-2 cases in legalization. https://github.com/llvm/llvm-project/issues/80671 -; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9,GFX9 %s -; xUN: llc -O0 -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9-O0,GFX9-O0 %s +; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9-G %s +; RUN: llc -O0 -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9-G-O0 %s define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-LABEL: v_sdiv_i128_vv: @@ -1223,6 +1222,1158 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) ; GFX9-O0-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-G-LABEL: v_sdiv_i128_vv: +; GFX9-G: ; %bb.0: ; %_udiv-special-cases +; GFX9-G-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-G-NEXT: v_ashrrev_i32_e32 v16, 31, v3 +; GFX9-G-NEXT: v_xor_b32_e32 v0, v16, v0 +; GFX9-G-NEXT: v_xor_b32_e32 v1, v16, v1 +; GFX9-G-NEXT: v_sub_co_u32_e32 v10, vcc, v0, v16 +; GFX9-G-NEXT: v_xor_b32_e32 v2, v16, v2 +; GFX9-G-NEXT: v_subb_co_u32_e32 v11, vcc, v1, v16, vcc +; GFX9-G-NEXT: v_ashrrev_i32_e32 v17, 31, v7 +; GFX9-G-NEXT: v_xor_b32_e32 v3, v16, v3 +; GFX9-G-NEXT: v_subb_co_u32_e32 v12, vcc, v2, v16, vcc +; GFX9-G-NEXT: v_subb_co_u32_e32 v13, vcc, v3, v16, vcc +; GFX9-G-NEXT: v_xor_b32_e32 v0, v17, v4 +; GFX9-G-NEXT: v_xor_b32_e32 v1, v17, v5 +; GFX9-G-NEXT: v_sub_co_u32_e32 v18, vcc, v0, v17 +; GFX9-G-NEXT: v_xor_b32_e32 v2, v17, v6 +; GFX9-G-NEXT: v_subb_co_u32_e32 v19, vcc, v1, v17, vcc +; GFX9-G-NEXT: v_xor_b32_e32 v3, v17, v7 +; GFX9-G-NEXT: v_subb_co_u32_e32 v4, vcc, v2, v17, vcc +; GFX9-G-NEXT: v_subb_co_u32_e32 v5, vcc, v3, v17, vcc +; GFX9-G-NEXT: v_or_b32_e32 v0, v18, v4 +; GFX9-G-NEXT: v_or_b32_e32 v1, v19, v5 +; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] +; GFX9-G-NEXT: v_or_b32_e32 v0, v10, v12 +; GFX9-G-NEXT: v_or_b32_e32 v1, v11, v13 +; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1] +; GFX9-G-NEXT: v_ffbh_u32_e32 v1, v18 +; GFX9-G-NEXT: v_ffbh_u32_e32 v0, v19 +; GFX9-G-NEXT: v_add_u32_e32 v1, 32, v1 +; GFX9-G-NEXT: v_ffbh_u32_e32 v2, v4 +; GFX9-G-NEXT: v_min_u32_e32 v0, v0, v1 +; GFX9-G-NEXT: v_ffbh_u32_e32 v1, v5 +; GFX9-G-NEXT: v_add_u32_e32 v2, 32, v2 +; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[4:5] +; GFX9-G-NEXT: v_add_u32_e32 v0, 64, v0 +; GFX9-G-NEXT: v_min_u32_e32 v1, v1, v2 +; GFX9-G-NEXT: v_ffbh_u32_e32 v2, v10 +; GFX9-G-NEXT: v_cndmask_b32_e64 v0, v1, v0, s[6:7] +; GFX9-G-NEXT: v_ffbh_u32_e32 v1, v11 +; GFX9-G-NEXT: v_add_u32_e32 v2, 32, v2 +; GFX9-G-NEXT: v_ffbh_u32_e32 v3, v12 +; GFX9-G-NEXT: v_min_u32_e32 v1, v1, v2 +; GFX9-G-NEXT: v_ffbh_u32_e32 v2, v13 +; GFX9-G-NEXT: v_add_u32_e32 v3, 32, v3 +; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[12:13] +; GFX9-G-NEXT: v_add_u32_e32 v1, 64, v1 +; GFX9-G-NEXT: v_min_u32_e32 v2, v2, v3 +; GFX9-G-NEXT: v_cndmask_b32_e64 v1, v2, v1, s[6:7] +; GFX9-G-NEXT: v_sub_co_u32_e64 v0, s[6:7], v0, v1 +; GFX9-G-NEXT: v_subb_co_u32_e64 v1, s[6:7], 0, 0, s[6:7] +; GFX9-G-NEXT: v_mov_b32_e32 v6, 0x7f +; GFX9-G-NEXT: v_subb_co_u32_e64 v2, s[6:7], 0, 0, s[6:7] +; GFX9-G-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-G-NEXT: v_subb_co_u32_e64 v3, s[6:7], 0, 0, s[6:7] +; GFX9-G-NEXT: v_cmp_gt_u64_e64 s[6:7], v[0:1], v[6:7] +; GFX9-G-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX9-G-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[6:7] +; GFX9-G-NEXT: v_cmp_lt_u64_e64 s[6:7], 0, v[2:3] +; GFX9-G-NEXT: v_or_b32_e32 v15, v1, v3 +; GFX9-G-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[6:7] +; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[2:3] +; GFX9-G-NEXT: s_mov_b64 s[8:9], 0 +; GFX9-G-NEXT: v_cndmask_b32_e64 v6, v7, v6, s[6:7] +; GFX9-G-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[4:5] +; GFX9-G-NEXT: v_or_b32_e32 v20, v7, v6 +; GFX9-G-NEXT: v_xor_b32_e32 v6, 0x7f, v0 +; GFX9-G-NEXT: v_or_b32_e32 v14, v6, v2 +; GFX9-G-NEXT: v_and_b32_e32 v6, 1, v20 +; GFX9-G-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6 +; GFX9-G-NEXT: v_cndmask_b32_e64 v6, v10, 0, vcc +; GFX9-G-NEXT: v_cndmask_b32_e64 v7, v11, 0, vcc +; GFX9-G-NEXT: v_cndmask_b32_e64 v8, v12, 0, vcc +; GFX9-G-NEXT: v_cndmask_b32_e64 v9, v13, 0, vcc +; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15] +; GFX9-G-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc +; GFX9-G-NEXT: v_or_b32_e32 v14, v20, v14 +; GFX9-G-NEXT: v_and_b32_e32 v14, 1, v14 +; GFX9-G-NEXT: v_cmp_ne_u32_e32 vcc, 0, v14 +; GFX9-G-NEXT: s_xor_b64 s[4:5], vcc, -1 +; GFX9-G-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] +; GFX9-G-NEXT: s_cbranch_execz .LBB0_6 +; GFX9-G-NEXT: ; %bb.1: ; %udiv-bb1 +; GFX9-G-NEXT: v_add_co_u32_e32 v20, vcc, 1, v0 +; GFX9-G-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v1, vcc +; GFX9-G-NEXT: v_addc_co_u32_e32 v22, vcc, 0, v2, vcc +; GFX9-G-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v3, vcc +; GFX9-G-NEXT: s_xor_b64 s[4:5], vcc, -1 +; GFX9-G-NEXT: v_sub_co_u32_e32 v8, vcc, 0x7f, v0 +; GFX9-G-NEXT: v_sub_u32_e32 v0, 64, v8 +; GFX9-G-NEXT: v_lshrrev_b64 v[0:1], v0, v[10:11] +; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], v8, v[12:13] +; GFX9-G-NEXT: v_subrev_u32_e32 v9, 64, v8 +; GFX9-G-NEXT: v_lshlrev_b64 v[6:7], v8, v[10:11] +; GFX9-G-NEXT: v_or_b32_e32 v2, v0, v2 +; GFX9-G-NEXT: v_or_b32_e32 v3, v1, v3 +; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], v9, v[10:11] +; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v8 +; GFX9-G-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v7, 0, v7, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX9-G-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8 +; GFX9-G-NEXT: v_cndmask_b32_e32 v8, v0, v12, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v9, v1, v13, vcc +; GFX9-G-NEXT: s_mov_b64 s[10:11], s[8:9] +; GFX9-G-NEXT: v_mov_b32_e32 v0, s8 +; GFX9-G-NEXT: v_mov_b32_e32 v1, s9 +; GFX9-G-NEXT: v_mov_b32_e32 v2, s10 +; GFX9-G-NEXT: v_mov_b32_e32 v3, s11 +; GFX9-G-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] +; GFX9-G-NEXT: s_xor_b64 s[12:13], exec, s[8:9] +; GFX9-G-NEXT: s_cbranch_execz .LBB0_5 +; GFX9-G-NEXT: ; %bb.2: ; %udiv-preheader +; GFX9-G-NEXT: v_sub_u32_e32 v2, 64, v20 +; GFX9-G-NEXT: v_lshrrev_b64 v[0:1], v20, v[10:11] +; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], v2, v[12:13] +; GFX9-G-NEXT: v_subrev_u32_e32 v24, 64, v20 +; GFX9-G-NEXT: v_lshrrev_b64 v[14:15], v20, v[12:13] +; GFX9-G-NEXT: v_or_b32_e32 v2, v0, v2 +; GFX9-G-NEXT: v_or_b32_e32 v3, v1, v3 +; GFX9-G-NEXT: v_lshrrev_b64 v[0:1], v24, v[12:13] +; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v20 +; GFX9-G-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v14, 0, v14, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v15, 0, v15, vcc +; GFX9-G-NEXT: v_add_co_u32_e32 v24, vcc, -1, v18 +; GFX9-G-NEXT: s_mov_b64 s[8:9], 0 +; GFX9-G-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v20 +; GFX9-G-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v19, vcc +; GFX9-G-NEXT: v_cndmask_b32_e64 v12, v0, v10, s[4:5] +; GFX9-G-NEXT: v_cndmask_b32_e64 v13, v1, v11, s[4:5] +; GFX9-G-NEXT: v_addc_co_u32_e32 v26, vcc, -1, v4, vcc +; GFX9-G-NEXT: s_mov_b64 s[10:11], s[8:9] +; GFX9-G-NEXT: v_mov_b32_e32 v0, s8 +; GFX9-G-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v5, vcc +; GFX9-G-NEXT: v_mov_b32_e32 v11, 0 +; GFX9-G-NEXT: v_mov_b32_e32 v1, s9 +; GFX9-G-NEXT: v_mov_b32_e32 v2, s10 +; GFX9-G-NEXT: v_mov_b32_e32 v3, s11 +; GFX9-G-NEXT: .LBB0_3: ; %udiv-do-while +; GFX9-G-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], 1, v[6:7] +; GFX9-G-NEXT: v_lshrrev_b32_e32 v10, 31, v7 +; GFX9-G-NEXT: v_or_b32_e32 v6, v0, v2 +; GFX9-G-NEXT: v_or_b32_e32 v7, v1, v3 +; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], 1, v[12:13] +; GFX9-G-NEXT: v_lshrrev_b32_e32 v12, 31, v9 +; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 1, v[14:15] +; GFX9-G-NEXT: v_or_b32_e32 v2, v2, v12 +; GFX9-G-NEXT: v_lshrrev_b32_e32 v14, 31, v13 +; GFX9-G-NEXT: v_sub_co_u32_e32 v12, vcc, v24, v2 +; GFX9-G-NEXT: v_or_b32_e32 v0, v0, v14 +; GFX9-G-NEXT: v_subb_co_u32_e32 v12, vcc, v25, v3, vcc +; GFX9-G-NEXT: v_subb_co_u32_e32 v12, vcc, v26, v0, vcc +; GFX9-G-NEXT: v_subb_co_u32_e32 v12, vcc, v27, v1, vcc +; GFX9-G-NEXT: v_ashrrev_i32_e32 v28, 31, v12 +; GFX9-G-NEXT: v_and_b32_e32 v12, v28, v18 +; GFX9-G-NEXT: v_sub_co_u32_e32 v12, vcc, v2, v12 +; GFX9-G-NEXT: v_and_b32_e32 v2, v28, v19 +; GFX9-G-NEXT: v_subb_co_u32_e32 v13, vcc, v3, v2, vcc +; GFX9-G-NEXT: v_and_b32_e32 v2, v28, v4 +; GFX9-G-NEXT: v_subb_co_u32_e32 v14, vcc, v0, v2, vcc +; GFX9-G-NEXT: v_and_b32_e32 v0, v28, v5 +; GFX9-G-NEXT: v_subb_co_u32_e32 v15, vcc, v1, v0, vcc +; GFX9-G-NEXT: v_add_co_u32_e32 v20, vcc, -1, v20 +; GFX9-G-NEXT: v_addc_co_u32_e32 v21, vcc, -1, v21, vcc +; GFX9-G-NEXT: v_addc_co_u32_e32 v22, vcc, -1, v22, vcc +; GFX9-G-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v23, vcc +; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9] +; GFX9-G-NEXT: v_or_b32_e32 v0, v20, v22 +; GFX9-G-NEXT: v_or_b32_e32 v1, v21, v23 +; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] +; GFX9-G-NEXT: v_or_b32_e32 v8, v8, v10 +; GFX9-G-NEXT: v_and_b32_e32 v10, 1, v28 +; GFX9-G-NEXT: v_mov_b32_e32 v0, v10 +; GFX9-G-NEXT: s_or_b64 s[8:9], vcc, s[8:9] +; GFX9-G-NEXT: v_mov_b32_e32 v1, v11 +; GFX9-G-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GFX9-G-NEXT: s_cbranch_execnz .LBB0_3 +; GFX9-G-NEXT: ; %bb.4: ; %Flow +; GFX9-G-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX9-G-NEXT: .LBB0_5: ; %Flow2 +; GFX9-G-NEXT: s_or_b64 exec, exec, s[12:13] +; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], 1, v[6:7] +; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9] +; GFX9-G-NEXT: v_lshrrev_b32_e32 v4, 31, v7 +; GFX9-G-NEXT: v_or_b32_e32 v8, v8, v4 +; GFX9-G-NEXT: v_or_b32_e32 v6, v0, v2 +; GFX9-G-NEXT: v_or_b32_e32 v7, v1, v3 +; GFX9-G-NEXT: .LBB0_6: ; %Flow3 +; GFX9-G-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX9-G-NEXT: v_xor_b32_e32 v3, v17, v16 +; GFX9-G-NEXT: v_xor_b32_e32 v0, v6, v3 +; GFX9-G-NEXT: v_xor_b32_e32 v1, v7, v3 +; GFX9-G-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v3 +; GFX9-G-NEXT: v_xor_b32_e32 v2, v8, v3 +; GFX9-G-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9-G-NEXT: v_xor_b32_e32 v4, v9, v3 +; GFX9-G-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc +; GFX9-G-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v3, vcc +; GFX9-G-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-G-O0-LABEL: v_sdiv_i128_vv: +; GFX9-G-O0: ; %bb.0: ; %_udiv-special-cases +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-G-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-G-O0-NEXT: ; implicit-def: $vgpr8 : SGPR spill to VGPR lane +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v0 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v3 +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v7 +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14_vgpr15_vgpr16 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v1 +; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0 +; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], 0x7f +; GFX9-G-O0-NEXT: ; kill: def $vgpr1_vgpr2 killed $vgpr4_vgpr5 killed $exec +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v7 +; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr1 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v2 +; GFX9-G-O0-NEXT: s_mov_b32 s6, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s6 +; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v12, v3, v8 +; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr1 killed $exec +; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr1_vgpr2 killed $exec +; GFX9-G-O0-NEXT: s_mov_b32 s6, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s6 +; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v10, v1, v2 +; GFX9-G-O0-NEXT: ; kill: def $vgpr1_vgpr2 killed $vgpr13_vgpr14 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v16 +; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr1 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v2 +; GFX9-G-O0-NEXT: s_mov_b32 s6, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s6 +; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v11, v3, v8 +; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr1 killed $exec +; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr1_vgpr2 killed $exec +; GFX9-G-O0-NEXT: s_mov_b32 s6, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s6 +; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v9, v1, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v1, v12, v1 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v4, v12, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v6 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v3, v10, v3 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v2, v10, v2 +; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v1, s[6:7], v1, v12 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[6:7], v4, v12, s[6:7] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[6:7], v3, v10, s[6:7] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v5, s[6:7], v2, v10, s[6:7] +; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2_vgpr3_vgpr4 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5 +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v16 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v7 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v5, v11, v5 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v8, v11, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v14 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v7, v9, v7 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v6, v9, v6 +; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v5, s[6:7], v5, v11 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v15, s[6:7], v8, v11, s[6:7] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v14, s[6:7], v7, v9, s[6:7] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v13, s[6:7], v6, v9, s[6:7] +; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6_vgpr7_vgpr8 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v13 +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_xor_b32_e64 v13, v11, v12 +; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_xor_b32_e64 v11, v11, v12 +; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_xor_b32_e64 v11, v9, v10 +; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_xor_b32_e64 v9, v9, v10 +; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v14 +; GFX9-G-O0-NEXT: v_or_b32_e64 v9, v9, v12 +; GFX9-G-O0-NEXT: v_or_b32_e64 v11, v10, v11 +; GFX9-G-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[6:7], v[9:10], v[11:12] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v14 +; GFX9-G-O0-NEXT: v_or_b32_e64 v9, v9, v12 +; GFX9-G-O0-NEXT: v_or_b32_e64 v11, v10, v11 +; GFX9-G-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[9:10], v[11:12] +; GFX9-G-O0-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[8:9], v[5:6] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v11 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v5 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, 32 +; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v7 +; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v6 +; GFX9-G-O0-NEXT: s_mov_b32 s10, 64 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10 +; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v5, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v9 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v5 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v7, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, 32 +; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8 +; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v7 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[8:9] +; GFX9-G-O0-NEXT: s_mov_b32 s16, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[9:10], v[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v12 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v7, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, 32 +; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8 +; GFX9-G-O0-NEXT: v_min_u32_e64 v6, v6, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s10 +; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v6, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v10 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v8, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, 32 +; GFX9-G-O0-NEXT: v_add_u32_e64 v8, v8, v9 +; GFX9-G-O0-NEXT: v_min_u32_e64 v6, v6, v8 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[8:9] +; GFX9-G-O0-NEXT: s_mov_b32 s15, 0 +; GFX9-G-O0-NEXT: s_mov_b32 s11, 0 +; GFX9-G-O0-NEXT: s_mov_b32 s14, 0 +; GFX9-G-O0-NEXT: s_mov_b32 s10, 0 +; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v6, s[8:9], v5, v6 +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s16 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s16 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[8:9], v5, v7, s[8:9] +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s14 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v9, s[8:9], v5, v8, s[8:9] +; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s10 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v5, v8, s[8:9] +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s4 +; GFX9-G-O0-NEXT: v_cmp_gt_u64_e64 s[10:11], v[12:13], v[14:15] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[12:13], v[14:15] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s12 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s13 +; GFX9-G-O0-NEXT: v_cmp_gt_u64_e64 s[12:13], v[10:11], v[12:13] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v10, v5, v10, s[12:13] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[10:11] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v10, v5, v10, s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[6:7] +; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v5, v10 +; GFX9-G-O0-NEXT: s_mov_b32 s7, 0x7f +; GFX9-G-O0-NEXT: s_mov_b32 s6, 0 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v6, v6, s7 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v7, v7, s6 +; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v6, v9 +; GFX9-G-O0-NEXT: v_or_b32_e64 v8, v7, v8 +; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[6:7], v[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v3 +; GFX9-G-O0-NEXT: v_and_b32_e32 v1, 1, v5 +; GFX9-G-O0-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v7 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v1, v1, v4, s[6:7] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v2, v3, s[6:7] +; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-G-O0-NEXT: v_and_b32_e32 v3, 1, v5 +; GFX9-G-O0-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[6:7] +; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v3 +; GFX9-G-O0-NEXT: ; kill: def $vgpr1_vgpr2 killed $vgpr1_vgpr2 def $vgpr1_vgpr2_vgpr3_vgpr4 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5] +; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v5, v6 +; GFX9-G-O0-NEXT: v_and_b32_e32 v5, 1, v5 +; GFX9-G-O0-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v5 +; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], -1 +; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7] +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], exec +; GFX9-G-O0-NEXT: v_writelane_b32 v0, s4, 0 +; GFX9-G-O0-NEXT: v_writelane_b32 v0, s5, 1 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7] +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-G-O0-NEXT: s_cbranch_execz .LBB0_3 +; GFX9-G-O0-NEXT: s_branch .LBB0_8 +; GFX9-G-O0-NEXT: .LBB0_1: ; %Flow +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_readlane_b32 s4, v0, 2 +; GFX9-G-O0-NEXT: v_readlane_b32 s5, v0, 3 +; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-G-O0-NEXT: ; %bb.2: ; %Flow +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4) +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_branch .LBB0_5 +; GFX9-G-O0-NEXT: .LBB0_3: ; %Flow2 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_readlane_b32 s4, v4, 0 +; GFX9-G-O0-NEXT: v_readlane_b32 s5, v4, 1 +; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_branch .LBB0_9 +; GFX9-G-O0-NEXT: .LBB0_4: ; %udiv-loop-exit +; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[10:11], v0, v[2:3] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[4:5] +; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr2 killed $exec +; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec +; GFX9-G-O0-NEXT: s_mov_b32 s4, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v6, v2, v3 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v16 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v17 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v12 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v11 +; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v7 +; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v1, v5 +; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v9 +; GFX9-G-O0-NEXT: v_or3_b32 v4, v4, v6, v7 +; GFX9-G-O0-NEXT: v_or3_b32 v2, v2, v3, v5 +; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v2 +; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_branch .LBB0_3 +; GFX9-G-O0-NEXT: .LBB0_5: ; %Flow1 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_readlane_b32 s4, v8, 4 +; GFX9-G-O0-NEXT: v_readlane_b32 s5, v8, 5 +; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_branch .LBB0_4 +; GFX9-G-O0-NEXT: .LBB0_6: ; %udiv-do-while +; GFX9-G-O0-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_readlane_b32 s6, v16, 6 +; GFX9-G-O0-NEXT: v_readlane_b32 s7, v16, 7 +; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5 +; GFX9-G-O0-NEXT: s_mov_b32 s8, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s8 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[21:22], v2, v[0:1] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s8 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v2, v[3:4] +; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr0 killed $exec +; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec +; GFX9-G-O0-NEXT: s_mov_b32 s9, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s9 +; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v3, v0, v1 +; GFX9-G-O0-NEXT: s_mov_b32 s9, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5 +; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v2, v3 +; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v0, v1 +; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr12_vgpr13 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15 +; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr0 killed $exec +; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec +; GFX9-G-O0-NEXT: s_mov_b32 s9, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s9 +; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v3, v0, v1 +; GFX9-G-O0-NEXT: s_mov_b32 s9, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v21 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v22 +; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v2, v3 +; GFX9-G-O0-NEXT: v_or_b32_e64 v9, v0, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[23:24], v0, v[2:3] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[12:13] +; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr2 killed $exec +; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec +; GFX9-G-O0-NEXT: s_mov_b32 s8, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s8 +; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v14, v2, v3 +; GFX9-G-O0-NEXT: s_mov_b32 s8, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v29, v31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v30, v32 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v33 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v34 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v29 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v30 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v23 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v24 +; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v15 +; GFX9-G-O0-NEXT: v_or_b32_e64 v13, v1, v13 +; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v21 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v22 +; GFX9-G-O0-NEXT: v_or3_b32 v12, v12, v14, v15 +; GFX9-G-O0-NEXT: v_or3_b32 v2, v2, v3, v13 +; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v2 +; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v11, s[8:9], v11, v4 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[8:9], v10, v9, s[8:9] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v8, v7, s[8:9] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[8:9], v6, v5, s[8:9] +; GFX9-G-O0-NEXT: s_mov_b32 s8, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8 +; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v8, v6, v10 +; GFX9-G-O0-NEXT: s_mov_b32 s8, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8 +; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v6, v6, v10 +; GFX9-G-O0-NEXT: s_mov_b32 s9, 1 +; GFX9-G-O0-NEXT: s_mov_b32 s8, 0 +; GFX9-G-O0-NEXT: v_and_b32_e64 v12, v8, s9 +; GFX9-G-O0-NEXT: v_and_b32_e64 v10, v8, s8 +; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s4 +; GFX9-G-O0-NEXT: ; kill: def $vgpr12_vgpr13 killed $vgpr12_vgpr13 def $vgpr12_vgpr13_vgpr14_vgpr15 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, v25 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v24, v26 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v27 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v28 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v23 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v24 +; GFX9-G-O0-NEXT: v_and_b32_e64 v11, v8, v11 +; GFX9-G-O0-NEXT: v_and_b32_e64 v10, v8, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v21 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v22 +; GFX9-G-O0-NEXT: v_and_b32_e64 v8, v6, v8 +; GFX9-G-O0-NEXT: v_and_b32_e64 v6, v6, v21 +; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[8:9], v4, v11 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[8:9], v9, v10, s[8:9] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v9, s[8:9], v7, v8, s[8:9] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v5, v6, s[8:9] +; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v17 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v18 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v19 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v20 +; GFX9-G-O0-NEXT: s_mov_b32 s8, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s12, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s11, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s10, -1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, s8 +; GFX9-G-O0-NEXT: v_add_co_u32_e64 v17, s[8:9], v11, v17 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s12 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v18, s[8:9], v10, v11, s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s11 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v20, s[8:9], v9, v10, s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s10 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v19, s[8:9], v8, v9, s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v17 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v18 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v20 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v19 +; GFX9-G-O0-NEXT: v_or_b32_e64 v17, v17, v20 +; GFX9-G-O0-NEXT: v_or_b32_e64 v19, v18, v19 +; GFX9-G-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, v19 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[17:18], v[19:20] +; GFX9-G-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v0 +; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v12 +; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[4:5] +; GFX9-G-O0-NEXT: v_writelane_b32 v16, s6, 2 +; GFX9-G-O0-NEXT: v_writelane_b32 v16, s7, 3 +; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[4:5] +; GFX9-G-O0-NEXT: v_writelane_b32 v16, s6, 6 +; GFX9-G-O0-NEXT: v_writelane_b32 v16, s7, 7 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX9-G-O0-NEXT: s_cbranch_execnz .LBB0_6 +; GFX9-G-O0-NEXT: s_branch .LBB0_1 +; GFX9-G-O0-NEXT: .LBB0_7: ; %udiv-preheader +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b32 s4, 64 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s4 +; GFX9-G-O0-NEXT: v_sub_u32_e64 v4, v13, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s4 +; GFX9-G-O0-NEXT: v_sub_u32_e64 v5, v5, v13 +; GFX9-G-O0-NEXT: s_mov_b32 s6, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s4 +; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v13, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s6 +; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v13, v6 +; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v13, v[21:22] +; GFX9-G-O0-NEXT: v_lshrrev_b64 v[26:27], v13, v[15:16] +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[24:25], v5, v[21:22] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v26 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v27 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, v24 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v25 +; GFX9-G-O0-NEXT: v_or_b32_e64 v14, v14, v23 +; GFX9-G-O0-NEXT: v_or_b32_e64 v13, v5, v13 +; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], 0 +; GFX9-G-O0-NEXT: v_lshrrev_b64 v[21:22], v4, v[21:22] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v21 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v22 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v14, s[4:5] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[4:5] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v16 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v14, s[6:7] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v13, v5, v13, s[6:7] +; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v13, v13, v14, s[4:5] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5] +; GFX9-G-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-G-O0-NEXT: ; kill: def $vgpr4_vgpr5 killed $vgpr4_vgpr5 def $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v17 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v18 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v19 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v20 +; GFX9-G-O0-NEXT: s_mov_b32 s4, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s10, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s7, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s6, -1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, s4 +; GFX9-G-O0-NEXT: v_add_co_u32_e64 v16, s[4:5], v16, v17 +; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s10 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v15, s[4:5], v15, v16, s[4:5] +; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s7 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v14, s[4:5], v14, v15, s[4:5] +; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v13, s[4:5], v13, v14, s[4:5] +; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], s[8:9] +; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[8:9] +; GFX9-G-O0-NEXT: v_writelane_b32 v12, s8, 6 +; GFX9-G-O0-NEXT: v_writelane_b32 v12, s9, 7 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s4 +; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_branch .LBB0_6 +; GFX9-G-O0-NEXT: .LBB0_8: ; %udiv-bb1 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0 +; GFX9-G-O0-NEXT: s_mov_b32 s6, 1 +; GFX9-G-O0-NEXT: s_mov_b32 s10, 0 +; GFX9-G-O0-NEXT: s_mov_b32 s9, 0 +; GFX9-G-O0-NEXT: s_mov_b32 s8, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s6 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3) +; GFX9-G-O0-NEXT: v_add_co_u32_e64 v5, s[6:7], v2, v5 +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1) +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[6:7], v4, v6, s[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s9 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v8, s[6:7], v3, v4, s[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v1, v3, s[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v7 +; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b32 s6, 0x7f +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s6 +; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[6:7], v1, v2 +; GFX9-G-O0-NEXT: s_mov_b32 s7, 64 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7 +; GFX9-G-O0-NEXT: v_sub_u32_e64 v3, v4, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7 +; GFX9-G-O0-NEXT: v_sub_u32_e64 v9, v1, v4 +; GFX9-G-O0-NEXT: s_mov_b32 s6, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7 +; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[8:9], v4, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s6 +; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, v1 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[1:2], v4, v[13:14] +; GFX9-G-O0-NEXT: v_lshrrev_b64 v[18:19], v9, v[13:14] +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[16:17], v4, v[11:12] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v18 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v19 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v16 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v17 +; GFX9-G-O0-NEXT: v_or_b32_e64 v10, v10, v15 +; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v9 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[13:14], v3, v[13:14] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[8:9] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v2, v3, s[8:9] +; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v14 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v9, v9, v10, s[8:9] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v12 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v9, v9, v10, s[6:7] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[6:7] +; GFX9-G-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v3 +; GFX9-G-O0-NEXT: ; kill: def $vgpr1_vgpr2 killed $vgpr1_vgpr2 def $vgpr1_vgpr2_vgpr3_vgpr4 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v10 +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], s[4:5] +; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v5, v8 +; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v6, v7 +; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s4 +; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[5:6], v[7:8] +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s11 +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], exec +; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5] +; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7] +; GFX9-G-O0-NEXT: v_writelane_b32 v0, s6, 4 +; GFX9-G-O0-NEXT: v_writelane_b32 v0, s7, 5 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-G-O0-NEXT: s_cbranch_execz .LBB0_5 +; GFX9-G-O0-NEXT: s_branch .LBB0_7 +; GFX9-G-O0-NEXT: .LBB0_9: ; %udiv-end +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21] +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v12 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v0, v0, v8 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v1, v1, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v10 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v2, v2, v6 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v3, v3, v5 +; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v0, s[4:5], v0, v8 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v1, s[4:5], v1, v7, s[4:5] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v2, s[4:5], v2, v6, s[4:5] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v3, s[4:5], v3, v5, s[4:5] +; GFX9-G-O0-NEXT: ; kill: killed $vgpr4 +; GFX9-G-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_nop 0 +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31] %div = sdiv i128 %lhs, %rhs ret i128 %div } @@ -2306,6 +3457,1043 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) ; GFX9-O0-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-G-LABEL: v_udiv_i128_vv: +; GFX9-G: ; %bb.0: ; %_udiv-special-cases +; GFX9-G-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-G-NEXT: v_or_b32_e32 v8, v4, v6 +; GFX9-G-NEXT: v_or_b32_e32 v9, v5, v7 +; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] +; GFX9-G-NEXT: v_or_b32_e32 v8, v0, v2 +; GFX9-G-NEXT: v_or_b32_e32 v9, v1, v3 +; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[8:9] +; GFX9-G-NEXT: v_ffbh_u32_e32 v9, v4 +; GFX9-G-NEXT: v_ffbh_u32_e32 v8, v5 +; GFX9-G-NEXT: v_add_u32_e32 v9, 32, v9 +; GFX9-G-NEXT: v_ffbh_u32_e32 v10, v6 +; GFX9-G-NEXT: v_min_u32_e32 v8, v8, v9 +; GFX9-G-NEXT: v_ffbh_u32_e32 v9, v7 +; GFX9-G-NEXT: v_add_u32_e32 v10, 32, v10 +; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[6:7] +; GFX9-G-NEXT: v_add_u32_e32 v8, 64, v8 +; GFX9-G-NEXT: v_min_u32_e32 v9, v9, v10 +; GFX9-G-NEXT: v_ffbh_u32_e32 v10, v0 +; GFX9-G-NEXT: v_cndmask_b32_e64 v8, v9, v8, s[6:7] +; GFX9-G-NEXT: v_ffbh_u32_e32 v9, v1 +; GFX9-G-NEXT: v_add_u32_e32 v10, 32, v10 +; GFX9-G-NEXT: v_ffbh_u32_e32 v11, v2 +; GFX9-G-NEXT: v_min_u32_e32 v9, v9, v10 +; GFX9-G-NEXT: v_ffbh_u32_e32 v10, v3 +; GFX9-G-NEXT: v_add_u32_e32 v11, 32, v11 +; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[2:3] +; GFX9-G-NEXT: v_add_u32_e32 v9, 64, v9 +; GFX9-G-NEXT: v_min_u32_e32 v10, v10, v11 +; GFX9-G-NEXT: v_cndmask_b32_e64 v9, v10, v9, s[6:7] +; GFX9-G-NEXT: v_sub_co_u32_e64 v12, s[6:7], v8, v9 +; GFX9-G-NEXT: v_subb_co_u32_e64 v13, s[6:7], 0, 0, s[6:7] +; GFX9-G-NEXT: v_mov_b32_e32 v8, 0x7f +; GFX9-G-NEXT: v_subb_co_u32_e64 v14, s[6:7], 0, 0, s[6:7] +; GFX9-G-NEXT: v_mov_b32_e32 v9, 0 +; GFX9-G-NEXT: v_subb_co_u32_e64 v15, s[6:7], 0, 0, s[6:7] +; GFX9-G-NEXT: v_cmp_gt_u64_e64 s[6:7], v[12:13], v[8:9] +; GFX9-G-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX9-G-NEXT: v_cndmask_b32_e64 v8, 0, 1, s[6:7] +; GFX9-G-NEXT: v_cmp_lt_u64_e64 s[6:7], 0, v[14:15] +; GFX9-G-NEXT: v_or_b32_e32 v17, v13, v15 +; GFX9-G-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[6:7] +; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[14:15] +; GFX9-G-NEXT: s_mov_b64 s[8:9], 0 +; GFX9-G-NEXT: v_cndmask_b32_e64 v8, v9, v8, s[6:7] +; GFX9-G-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[4:5] +; GFX9-G-NEXT: v_or_b32_e32 v18, v9, v8 +; GFX9-G-NEXT: v_xor_b32_e32 v8, 0x7f, v12 +; GFX9-G-NEXT: v_or_b32_e32 v16, v8, v14 +; GFX9-G-NEXT: v_and_b32_e32 v8, 1, v18 +; GFX9-G-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8 +; GFX9-G-NEXT: v_cndmask_b32_e64 v10, v0, 0, vcc +; GFX9-G-NEXT: v_cndmask_b32_e64 v11, v1, 0, vcc +; GFX9-G-NEXT: v_cndmask_b32_e64 v8, v2, 0, vcc +; GFX9-G-NEXT: v_cndmask_b32_e64 v9, v3, 0, vcc +; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17] +; GFX9-G-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc +; GFX9-G-NEXT: v_or_b32_e32 v16, v18, v16 +; GFX9-G-NEXT: v_and_b32_e32 v16, 1, v16 +; GFX9-G-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16 +; GFX9-G-NEXT: s_xor_b64 s[4:5], vcc, -1 +; GFX9-G-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] +; GFX9-G-NEXT: s_cbranch_execz .LBB1_6 +; GFX9-G-NEXT: ; %bb.1: ; %udiv-bb1 +; GFX9-G-NEXT: v_add_co_u32_e32 v18, vcc, 1, v12 +; GFX9-G-NEXT: v_addc_co_u32_e32 v19, vcc, 0, v13, vcc +; GFX9-G-NEXT: v_addc_co_u32_e32 v20, vcc, 0, v14, vcc +; GFX9-G-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v15, vcc +; GFX9-G-NEXT: s_xor_b64 s[4:5], vcc, -1 +; GFX9-G-NEXT: v_sub_co_u32_e32 v16, vcc, 0x7f, v12 +; GFX9-G-NEXT: v_sub_u32_e32 v8, 64, v16 +; GFX9-G-NEXT: v_lshrrev_b64 v[8:9], v8, v[0:1] +; GFX9-G-NEXT: v_lshlrev_b64 v[10:11], v16, v[2:3] +; GFX9-G-NEXT: v_subrev_u32_e32 v14, 64, v16 +; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], v16, v[0:1] +; GFX9-G-NEXT: v_or_b32_e32 v10, v8, v10 +; GFX9-G-NEXT: v_or_b32_e32 v11, v9, v11 +; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], v14, v[0:1] +; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16 +; GFX9-G-NEXT: s_mov_b64 s[10:11], s[8:9] +; GFX9-G-NEXT: v_cndmask_b32_e32 v14, 0, v12, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v15, 0, v13, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc +; GFX9-G-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16 +; GFX9-G-NEXT: v_mov_b32_e32 v13, s11 +; GFX9-G-NEXT: v_cndmask_b32_e32 v8, v8, v2, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v9, v9, v3, vcc +; GFX9-G-NEXT: v_mov_b32_e32 v11, s9 +; GFX9-G-NEXT: v_mov_b32_e32 v10, s8 +; GFX9-G-NEXT: v_mov_b32_e32 v12, s10 +; GFX9-G-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] +; GFX9-G-NEXT: s_xor_b64 s[12:13], exec, s[8:9] +; GFX9-G-NEXT: s_cbranch_execz .LBB1_5 +; GFX9-G-NEXT: ; %bb.2: ; %udiv-preheader +; GFX9-G-NEXT: v_sub_u32_e32 v12, 64, v18 +; GFX9-G-NEXT: v_subrev_u32_e32 v22, 64, v18 +; GFX9-G-NEXT: v_lshrrev_b64 v[10:11], v18, v[0:1] +; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], v12, v[2:3] +; GFX9-G-NEXT: v_lshrrev_b64 v[16:17], v18, v[2:3] +; GFX9-G-NEXT: v_lshrrev_b64 v[2:3], v22, v[2:3] +; GFX9-G-NEXT: v_or_b32_e32 v10, v10, v12 +; GFX9-G-NEXT: v_or_b32_e32 v11, v11, v13 +; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v18 +; GFX9-G-NEXT: s_mov_b64 s[8:9], 0 +; GFX9-G-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc +; GFX9-G-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc +; GFX9-G-NEXT: v_add_co_u32_e32 v22, vcc, -1, v4 +; GFX9-G-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v5, vcc +; GFX9-G-NEXT: s_mov_b64 s[10:11], s[8:9] +; GFX9-G-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v18 +; GFX9-G-NEXT: v_addc_co_u32_e32 v24, vcc, -1, v6, vcc +; GFX9-G-NEXT: v_mov_b32_e32 v13, s11 +; GFX9-G-NEXT: v_cndmask_b32_e64 v2, v2, v0, s[4:5] +; GFX9-G-NEXT: v_cndmask_b32_e64 v3, v3, v1, s[4:5] +; GFX9-G-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v7, vcc +; GFX9-G-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-G-NEXT: v_mov_b32_e32 v11, s9 +; GFX9-G-NEXT: v_mov_b32_e32 v10, s8 +; GFX9-G-NEXT: v_mov_b32_e32 v12, s10 +; GFX9-G-NEXT: .LBB1_3: ; %udiv-do-while +; GFX9-G-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], 1, v[14:15] +; GFX9-G-NEXT: v_lshrrev_b32_e32 v0, 31, v15 +; GFX9-G-NEXT: v_or_b32_e32 v14, v10, v12 +; GFX9-G-NEXT: v_or_b32_e32 v15, v11, v13 +; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], 1, v[16:17] +; GFX9-G-NEXT: v_lshlrev_b64 v[10:11], 1, v[2:3] +; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 31, v3 +; GFX9-G-NEXT: v_or_b32_e32 v12, v12, v2 +; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 31, v9 +; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9] +; GFX9-G-NEXT: v_or_b32_e32 v2, v10, v2 +; GFX9-G-NEXT: v_or_b32_e32 v8, v8, v0 +; GFX9-G-NEXT: v_sub_co_u32_e32 v0, vcc, v22, v2 +; GFX9-G-NEXT: v_subb_co_u32_e32 v0, vcc, v23, v11, vcc +; GFX9-G-NEXT: v_subb_co_u32_e32 v0, vcc, v24, v12, vcc +; GFX9-G-NEXT: v_subb_co_u32_e32 v0, vcc, v25, v13, vcc +; GFX9-G-NEXT: v_add_co_u32_e64 v18, s[4:5], -1, v18 +; GFX9-G-NEXT: v_ashrrev_i32_e32 v3, 31, v0 +; GFX9-G-NEXT: v_addc_co_u32_e64 v19, s[4:5], -1, v19, s[4:5] +; GFX9-G-NEXT: v_and_b32_e32 v10, v3, v4 +; GFX9-G-NEXT: v_addc_co_u32_e64 v20, s[4:5], -1, v20, s[4:5] +; GFX9-G-NEXT: v_and_b32_e32 v16, v3, v5 +; GFX9-G-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v10 +; GFX9-G-NEXT: v_addc_co_u32_e64 v21, s[4:5], -1, v21, s[4:5] +; GFX9-G-NEXT: v_and_b32_e32 v0, 1, v3 +; GFX9-G-NEXT: v_and_b32_e32 v17, v3, v6 +; GFX9-G-NEXT: v_and_b32_e32 v26, v3, v7 +; GFX9-G-NEXT: v_subb_co_u32_e32 v3, vcc, v11, v16, vcc +; GFX9-G-NEXT: v_or_b32_e32 v10, v18, v20 +; GFX9-G-NEXT: v_or_b32_e32 v11, v19, v21 +; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[10:11] +; GFX9-G-NEXT: v_subb_co_u32_e32 v16, vcc, v12, v17, vcc +; GFX9-G-NEXT: v_mov_b32_e32 v11, v1 +; GFX9-G-NEXT: v_subb_co_u32_e32 v17, vcc, v13, v26, vcc +; GFX9-G-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9] +; GFX9-G-NEXT: v_mov_b32_e32 v10, v0 +; GFX9-G-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GFX9-G-NEXT: s_cbranch_execnz .LBB1_3 +; GFX9-G-NEXT: ; %bb.4: ; %Flow +; GFX9-G-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX9-G-NEXT: .LBB1_5: ; %Flow2 +; GFX9-G-NEXT: s_or_b64 exec, exec, s[12:13] +; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 1, v[14:15] +; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9] +; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 31, v15 +; GFX9-G-NEXT: v_or_b32_e32 v8, v8, v2 +; GFX9-G-NEXT: v_or_b32_e32 v10, v10, v0 +; GFX9-G-NEXT: v_or_b32_e32 v11, v11, v1 +; GFX9-G-NEXT: .LBB1_6: ; %Flow3 +; GFX9-G-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX9-G-NEXT: v_mov_b32_e32 v0, v10 +; GFX9-G-NEXT: v_mov_b32_e32 v1, v11 +; GFX9-G-NEXT: v_mov_b32_e32 v2, v8 +; GFX9-G-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-G-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-G-O0-LABEL: v_udiv_i128_vv: +; GFX9-G-O0: ; %bb.0: ; %_udiv-special-cases +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-G-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-G-O0-NEXT: ; implicit-def: $vgpr8 : SGPR spill to VGPR lane +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v0 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v3 +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v4 +; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_nop 0 +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v5 +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v7 +; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6_vgpr7_vgpr8 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v9 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v14 +; GFX9-G-O0-NEXT: v_or_b32_e64 v9, v9, v12 +; GFX9-G-O0-NEXT: v_or_b32_e64 v11, v10, v11 +; GFX9-G-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[6:7], v[9:10], v[11:12] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v14 +; GFX9-G-O0-NEXT: v_or_b32_e64 v9, v9, v12 +; GFX9-G-O0-NEXT: v_or_b32_e64 v11, v10, v11 +; GFX9-G-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[9:10], v[11:12] +; GFX9-G-O0-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[8:9], v[5:6] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v11 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v5 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, 32 +; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v7 +; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v6 +; GFX9-G-O0-NEXT: s_mov_b32 s10, 64 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10 +; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v5, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v9 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v5 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v7, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, 32 +; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8 +; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v7 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[8:9] +; GFX9-G-O0-NEXT: s_mov_b32 s14, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[9:10], v[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v12 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v7, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, 32 +; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8 +; GFX9-G-O0-NEXT: v_min_u32_e64 v6, v6, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s10 +; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v6, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v10 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6 +; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v8, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, 32 +; GFX9-G-O0-NEXT: v_add_u32_e64 v8, v8, v9 +; GFX9-G-O0-NEXT: v_min_u32_e64 v6, v6, v8 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[8:9] +; GFX9-G-O0-NEXT: s_mov_b32 s13, 0 +; GFX9-G-O0-NEXT: s_mov_b32 s11, 0 +; GFX9-G-O0-NEXT: s_mov_b32 s12, 0 +; GFX9-G-O0-NEXT: s_mov_b32 s10, 0 +; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v6, s[8:9], v5, v6 +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s14 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[8:9], v5, v7, s[8:9] +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s12 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v9, s[8:9], v5, v8, s[8:9] +; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s10 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v5, v8, s[8:9] +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], 0x7f +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s4 +; GFX9-G-O0-NEXT: v_cmp_gt_u64_e64 s[10:11], v[12:13], v[14:15] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[12:13], v[14:15] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s12 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s13 +; GFX9-G-O0-NEXT: v_cmp_gt_u64_e64 s[12:13], v[10:11], v[12:13] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v10, v5, v10, s[12:13] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[10:11] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v10, v5, v10, s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[6:7] +; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v5, v10 +; GFX9-G-O0-NEXT: s_mov_b32 s7, 0x7f +; GFX9-G-O0-NEXT: s_mov_b32 s6, 0 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v6, v6, s7 +; GFX9-G-O0-NEXT: v_xor_b32_e64 v7, v7, s6 +; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v6, v9 +; GFX9-G-O0-NEXT: v_or_b32_e64 v8, v7, v8 +; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[6:7], v[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v3 +; GFX9-G-O0-NEXT: v_and_b32_e32 v1, 1, v5 +; GFX9-G-O0-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v7 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v1, v1, v4, s[6:7] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v2, v3, s[6:7] +; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-G-O0-NEXT: v_and_b32_e32 v3, 1, v5 +; GFX9-G-O0-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[6:7] +; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v3 +; GFX9-G-O0-NEXT: ; kill: def $vgpr1_vgpr2 killed $vgpr1_vgpr2 def $vgpr1_vgpr2_vgpr3_vgpr4 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5] +; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v5, v6 +; GFX9-G-O0-NEXT: v_and_b32_e32 v5, 1, v5 +; GFX9-G-O0-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v5 +; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], -1 +; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7] +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], exec +; GFX9-G-O0-NEXT: v_writelane_b32 v0, s4, 0 +; GFX9-G-O0-NEXT: v_writelane_b32 v0, s5, 1 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7] +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-G-O0-NEXT: s_cbranch_execz .LBB1_3 +; GFX9-G-O0-NEXT: s_branch .LBB1_8 +; GFX9-G-O0-NEXT: .LBB1_1: ; %Flow +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_readlane_b32 s4, v0, 2 +; GFX9-G-O0-NEXT: v_readlane_b32 s5, v0, 3 +; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-G-O0-NEXT: ; %bb.2: ; %Flow +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4) +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_branch .LBB1_5 +; GFX9-G-O0-NEXT: .LBB1_3: ; %Flow2 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_readlane_b32 s4, v4, 0 +; GFX9-G-O0-NEXT: v_readlane_b32 s5, v4, 1 +; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_branch .LBB1_9 +; GFX9-G-O0-NEXT: .LBB1_4: ; %udiv-loop-exit +; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[10:11], v0, v[2:3] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[4:5] +; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr2 killed $exec +; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec +; GFX9-G-O0-NEXT: s_mov_b32 s4, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v6, v2, v3 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v16 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v17 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v12 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v11 +; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v7 +; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v1, v5 +; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v9 +; GFX9-G-O0-NEXT: v_or3_b32 v4, v4, v6, v7 +; GFX9-G-O0-NEXT: v_or3_b32 v2, v2, v3, v5 +; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v2 +; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_branch .LBB1_3 +; GFX9-G-O0-NEXT: .LBB1_5: ; %Flow1 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_readlane_b32 s4, v8, 4 +; GFX9-G-O0-NEXT: v_readlane_b32 s5, v8, 5 +; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_branch .LBB1_4 +; GFX9-G-O0-NEXT: .LBB1_6: ; %udiv-do-while +; GFX9-G-O0-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_readlane_b32 s6, v16, 6 +; GFX9-G-O0-NEXT: v_readlane_b32 s7, v16, 7 +; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5 +; GFX9-G-O0-NEXT: s_mov_b32 s8, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s8 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[21:22], v2, v[0:1] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s8 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v2, v[3:4] +; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr0 killed $exec +; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec +; GFX9-G-O0-NEXT: s_mov_b32 s9, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s9 +; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v3, v0, v1 +; GFX9-G-O0-NEXT: s_mov_b32 s9, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5 +; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v2, v3 +; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v0, v1 +; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr12_vgpr13 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15 +; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr0 killed $exec +; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec +; GFX9-G-O0-NEXT: s_mov_b32 s9, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s9 +; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v3, v0, v1 +; GFX9-G-O0-NEXT: s_mov_b32 s9, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v21 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v22 +; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v2, v3 +; GFX9-G-O0-NEXT: v_or_b32_e64 v9, v0, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[23:24], v0, v[2:3] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[12:13] +; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr2 killed $exec +; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec +; GFX9-G-O0-NEXT: s_mov_b32 s8, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s8 +; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v14, v2, v3 +; GFX9-G-O0-NEXT: s_mov_b32 s8, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v29, v31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v30, v32 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v33 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v34 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v29 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v30 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v23 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v24 +; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v15 +; GFX9-G-O0-NEXT: v_or_b32_e64 v13, v1, v13 +; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v21 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v22 +; GFX9-G-O0-NEXT: v_or3_b32 v12, v12, v14, v15 +; GFX9-G-O0-NEXT: v_or3_b32 v2, v2, v3, v13 +; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v2 +; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v11, s[8:9], v11, v4 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[8:9], v10, v9, s[8:9] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v8, v7, s[8:9] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[8:9], v6, v5, s[8:9] +; GFX9-G-O0-NEXT: s_mov_b32 s8, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8 +; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v8, v6, v10 +; GFX9-G-O0-NEXT: s_mov_b32 s8, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8 +; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v6, v6, v10 +; GFX9-G-O0-NEXT: s_mov_b32 s9, 1 +; GFX9-G-O0-NEXT: s_mov_b32 s8, 0 +; GFX9-G-O0-NEXT: v_and_b32_e64 v12, v8, s9 +; GFX9-G-O0-NEXT: v_and_b32_e64 v10, v8, s8 +; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s4 +; GFX9-G-O0-NEXT: ; kill: def $vgpr12_vgpr13 killed $vgpr12_vgpr13 def $vgpr12_vgpr13_vgpr14_vgpr15 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, v25 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v24, v26 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v27 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v28 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v23 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v24 +; GFX9-G-O0-NEXT: v_and_b32_e64 v11, v8, v11 +; GFX9-G-O0-NEXT: v_and_b32_e64 v10, v8, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v21 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v22 +; GFX9-G-O0-NEXT: v_and_b32_e64 v8, v6, v8 +; GFX9-G-O0-NEXT: v_and_b32_e64 v6, v6, v21 +; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[8:9], v4, v11 +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[8:9], v9, v10, s[8:9] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v9, s[8:9], v7, v8, s[8:9] +; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v5, v6, s[8:9] +; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v17 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v18 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v19 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v20 +; GFX9-G-O0-NEXT: s_mov_b32 s8, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s12, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s11, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s10, -1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, s8 +; GFX9-G-O0-NEXT: v_add_co_u32_e64 v17, s[8:9], v11, v17 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s12 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v18, s[8:9], v10, v11, s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s11 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v20, s[8:9], v9, v10, s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s10 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v19, s[8:9], v8, v9, s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v17 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v18 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v20 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v19 +; GFX9-G-O0-NEXT: v_or_b32_e64 v17, v17, v20 +; GFX9-G-O0-NEXT: v_or_b32_e64 v19, v18, v19 +; GFX9-G-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, v19 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, s4 +; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[17:18], v[19:20] +; GFX9-G-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v0 +; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v12 +; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[4:5] +; GFX9-G-O0-NEXT: v_writelane_b32 v16, s6, 2 +; GFX9-G-O0-NEXT: v_writelane_b32 v16, s7, 3 +; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[4:5] +; GFX9-G-O0-NEXT: v_writelane_b32 v16, s6, 6 +; GFX9-G-O0-NEXT: v_writelane_b32 v16, s7, 7 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX9-G-O0-NEXT: s_cbranch_execnz .LBB1_6 +; GFX9-G-O0-NEXT: s_branch .LBB1_1 +; GFX9-G-O0-NEXT: .LBB1_7: ; %udiv-preheader +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b32 s4, 64 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s4 +; GFX9-G-O0-NEXT: v_sub_u32_e64 v4, v13, v4 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s4 +; GFX9-G-O0-NEXT: v_sub_u32_e64 v5, v5, v13 +; GFX9-G-O0-NEXT: s_mov_b32 s6, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s4 +; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v13, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s6 +; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v13, v6 +; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v13, v[21:22] +; GFX9-G-O0-NEXT: v_lshrrev_b64 v[26:27], v13, v[15:16] +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[24:25], v5, v[21:22] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v26 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v27 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, v24 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v25 +; GFX9-G-O0-NEXT: v_or_b32_e64 v14, v14, v23 +; GFX9-G-O0-NEXT: v_or_b32_e64 v13, v5, v13 +; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], 0 +; GFX9-G-O0-NEXT: v_lshrrev_b64 v[21:22], v4, v[21:22] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v21 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v22 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v14, s[4:5] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[4:5] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v15 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v16 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v14, s[6:7] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v13, v5, v13, s[6:7] +; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v13, v13, v14, s[4:5] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5] +; GFX9-G-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-G-O0-NEXT: ; kill: def $vgpr4_vgpr5 killed $vgpr4_vgpr5 def $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v14 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v17 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v18 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v19 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v20 +; GFX9-G-O0-NEXT: s_mov_b32 s4, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s10, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s7, -1 +; GFX9-G-O0-NEXT: s_mov_b32 s6, -1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, s4 +; GFX9-G-O0-NEXT: v_add_co_u32_e64 v16, s[4:5], v16, v17 +; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s10 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v15, s[4:5], v15, v16, s[4:5] +; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s7 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v14, s[4:5], v14, v15, s[4:5] +; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v13, s[4:5], v13, v14, s[4:5] +; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], s[8:9] +; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[8:9] +; GFX9-G-O0-NEXT: v_writelane_b32 v12, s8, 6 +; GFX9-G-O0-NEXT: v_writelane_b32 v12, s9, 7 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s4 +; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_branch .LBB1_6 +; GFX9-G-O0-NEXT: .LBB1_8: ; %udiv-bb1 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0 +; GFX9-G-O0-NEXT: s_mov_b32 s6, 1 +; GFX9-G-O0-NEXT: s_mov_b32 s10, 0 +; GFX9-G-O0-NEXT: s_mov_b32 s9, 0 +; GFX9-G-O0-NEXT: s_mov_b32 s8, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s6 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3) +; GFX9-G-O0-NEXT: v_add_co_u32_e64 v5, s[6:7], v2, v5 +; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10 +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1) +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[6:7], v4, v6, s[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s9 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v8, s[6:7], v3, v4, s[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v1, v3, s[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v7 +; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b32 s6, 0x7f +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s6 +; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[6:7], v1, v2 +; GFX9-G-O0-NEXT: s_mov_b32 s7, 64 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7 +; GFX9-G-O0-NEXT: v_sub_u32_e64 v3, v4, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7 +; GFX9-G-O0-NEXT: v_sub_u32_e64 v9, v1, v4 +; GFX9-G-O0-NEXT: s_mov_b32 s6, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7 +; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[8:9], v4, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s6 +; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, v1 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[1:2], v4, v[13:14] +; GFX9-G-O0-NEXT: v_lshrrev_b64 v[18:19], v9, v[13:14] +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[16:17], v4, v[11:12] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v18 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v19 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v16 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v17 +; GFX9-G-O0-NEXT: v_or_b32_e64 v10, v10, v15 +; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v9 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[13:14], v3, v[13:14] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, 0 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[8:9] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v2, v3, s[8:9] +; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v13 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v14 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v9, v9, v10, s[8:9] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[8:9] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v11 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v12 +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v9, v9, v10, s[6:7] +; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[6:7] +; GFX9-G-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v3 +; GFX9-G-O0-NEXT: ; kill: def $vgpr1_vgpr2 killed $vgpr1_vgpr2 def $vgpr1_vgpr2_vgpr3_vgpr4 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v10 +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], s[4:5] +; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v5, v8 +; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v6, v7 +; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s4 +; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[5:6], v[7:8] +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s8 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s9 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s10 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s11 +; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], exec +; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5] +; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7] +; GFX9-G-O0-NEXT: v_writelane_b32 v0, s6, 4 +; GFX9-G-O0-NEXT: v_writelane_b32 v0, s7, 5 +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-G-O0-NEXT: s_cbranch_execz .LBB1_5 +; GFX9-G-O0-NEXT: s_branch .LBB1_7 +; GFX9-G-O0-NEXT: .LBB1_9: ; %udiv-end +; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v7 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v8 +; GFX9-G-O0-NEXT: ; kill: killed $vgpr4 +; GFX9-G-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_nop 0 +; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload +; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31] %div = udiv i128 %lhs, %rhs ret i128 %div } @@ -2388,6 +4576,66 @@ define i128 @v_sdiv_i128_v_pow2k(i128 %lhs) { ; GFX9-O0-NEXT: v_lshrrev_b64 v[3:4], s4, v[3:4] ; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr3_vgpr4 killed $exec ; GFX9-O0-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-G-LABEL: v_sdiv_i128_v_pow2k: +; GFX9-G: ; %bb.0: +; GFX9-G-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-G-NEXT: v_ashrrev_i32_e32 v4, 31, v3 +; GFX9-G-NEXT: v_mov_b32_e32 v5, v4 +; GFX9-G-NEXT: v_lshrrev_b64 v[4:5], 31, v[4:5] +; GFX9-G-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4 +; GFX9-G-NEXT: v_addc_co_u32_e32 v4, vcc, v1, v5, vcc +; GFX9-G-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc +; GFX9-G-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v3, vcc +; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 31, v[1:2] +; GFX9-G-NEXT: v_lshrrev_b32_e32 v3, 1, v4 +; GFX9-G-NEXT: v_or_b32_e32 v0, v3, v0 +; GFX9-G-NEXT: v_ashrrev_i32_e32 v3, 31, v2 +; GFX9-G-NEXT: v_ashrrev_i32_e32 v2, 1, v2 +; GFX9-G-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-G-O0-LABEL: v_sdiv_i128_v_pow2k: +; GFX9-G-O0: ; %bb.0: +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4 +; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v0, v0, v3 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v0 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4 +; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v0, v[5:6] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v7 +; GFX9-G-O0-NEXT: s_mov_b32 s8, 0 +; GFX9-G-O0-NEXT: s_mov_b32 s5, 0 +; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v4, v5 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v1, s[6:7], v1, v0, s[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v2, v0, s[6:7] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5 +; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v4, s[6:7], v3, v0, s[6:7] +; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v4 +; GFX9-G-O0-NEXT: s_mov_b32 s5, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5 +; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v0, v0, v1 +; GFX9-G-O0-NEXT: s_mov_b32 s5, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[5:6], v2, v[5:6] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v6 +; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v3 +; GFX9-G-O0-NEXT: v_or_b32_e64 v1, v1, v2 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v3, v2, v4 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v2, v2, v4 +; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31] %div = sdiv i128 %lhs, 8589934592 ret i128 %div } @@ -2434,10 +4682,42 @@ define i128 @v_udiv_i128_v_pow2k(i128 %lhs) { ; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v3, 0 ; GFX9-O0-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-G-LABEL: v_udiv_i128_v_pow2k: +; GFX9-G: ; %bb.0: +; GFX9-G-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-G-NEXT: v_mov_b32_e32 v4, v1 +; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 31, v[2:3] +; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 1, v4 +; GFX9-G-NEXT: v_or_b32_e32 v0, v2, v0 +; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 1, v3 +; GFX9-G-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-G-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-G-O0-LABEL: v_udiv_i128_v_pow2k: +; GFX9-G-O0: ; %bb.0: +; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4 +; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v0, v0, v1 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s4 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 31 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-G-O0-NEXT: v_lshlrev_b64 v[5:6], v2, v[4:5] +; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v6 +; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v4 +; GFX9-G-O0-NEXT: v_or_b32_e64 v1, v1, v2 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 1 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v2, v3 +; GFX9-G-O0-NEXT: s_mov_b32 s4, 0 +; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s4 +; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31] %div = udiv i128 %lhs, 8589934592 ret i128 %div } - -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; GFX9-SDAG: {{.*}} -; GFX9-SDAG-O0: {{.*}} -- GitLab From af34ac4508adcb5b54e62947d7caf7dd40227b16 Mon Sep 17 00:00:00 2001 From: Sirraide Date: Fri, 29 Mar 2024 17:42:09 +0100 Subject: [PATCH 0008/1690] [Clang] [Sema] No longer diagnose type definitions in `offsetof` in C23 mode (#84169) This is now allowed in C23; continue to diagnose it in earlier language modes as before, but now as a C23 extension rather than a GNU extension. This fixes #83658. --- clang/docs/ReleaseNotes.rst | 3 +++ .../include/clang/Basic/DiagnosticSemaKinds.td | 4 ++-- clang/lib/Sema/SemaDecl.cpp | 4 +++- clang/test/C/C2x/n2350.c | 10 +++++----- clang/test/C/drs/dr4xx.c | 17 ++++++++++------- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index c193c20fae5c..37b843915a0d 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -307,6 +307,9 @@ Improvements to Clang's diagnostics - ``-Wmicrosoft``, ``-Wgnu``, or ``-pedantic`` is now required to diagnose C99 flexible array members in a union or alone in a struct. Fixes GH#84565. +- Clang now no longer diagnoses type definitions in ``offsetof`` in C23 mode. + Fixes #GH83658. + Improvements to Clang's time-trace ---------------------------------- diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 51af81bf1f6f..fbc9c04dfba9 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -1748,8 +1748,8 @@ def err_type_defined_in_condition : Error< def err_type_defined_in_enum : Error< "%0 cannot be defined in an enumeration">; def ext_type_defined_in_offsetof : Extension< - "defining a type within '%select{__builtin_offsetof|offsetof}0' is a Clang " - "extension">, InGroup; + "defining a type within '%select{__builtin_offsetof|offsetof}0' is a C23 " + "extension">, InGroup; def note_pure_virtual_function : Note< "unimplemented pure virtual method %0 in %1">; diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 503f1f6f53a2..5027deda0d7e 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -18170,7 +18170,9 @@ CreateNewDecl: cast_or_null(PrevDecl)); } - if (OOK != OOK_Outside && TUK == TUK_Definition && !getLangOpts().CPlusPlus) + // Only C23 and later allow defining new types in 'offsetof()'. + if (OOK != OOK_Outside && TUK == TUK_Definition && !getLangOpts().CPlusPlus && + !getLangOpts().C23) Diag(New->getLocation(), diag::ext_type_defined_in_offsetof) << (OOK == OOK_Macro) << New->getSourceRange(); diff --git a/clang/test/C/C2x/n2350.c b/clang/test/C/C2x/n2350.c index 2f738488a374..af0ca6d79be5 100644 --- a/clang/test/C/C2x/n2350.c +++ b/clang/test/C/C2x/n2350.c @@ -5,7 +5,7 @@ // RUN: %clang_cc1 -fsyntax-only -pedantic -Wno-comment -std=c99 -verify %s // RUN: %clang_cc1 -fsyntax-only -pedantic -Wno-comment -std=c11 -verify %s // RUN: %clang_cc1 -fsyntax-only -pedantic -Wno-comment -std=c17 -verify %s -// RUN: %clang_cc1 -fsyntax-only -pedantic -Wno-comment -std=c2x -verify %s +// RUN: %clang_cc1 -fsyntax-only -pedantic -Wno-comment -std=c2x -verify=silent %s // silent-no-diagnostics @@ -13,10 +13,10 @@ // https://www.open-std.org/jtc1/sc22/wg14/www/docs/n2350.htm int simple(void) { return __builtin_offsetof(struct A // cpp-error {{'A' cannot be defined in a type specifier}} \ - expected-warning {{defining a type within '__builtin_offsetof' is a Clang extension}} + expected-warning {{defining a type within '__builtin_offsetof' is a C23 extension}} { int a; - struct B // expected-warning {{defining a type within '__builtin_offsetof' is a Clang extension}} + struct B // expected-warning {{defining a type within '__builtin_offsetof' is a C23 extension}} { int c; int d; @@ -26,7 +26,7 @@ int simple(void) { int anonymous_struct(void) { return __builtin_offsetof(struct // cpp-error-re {{'(unnamed struct at {{.*}})' cannot be defined in a type specifier}} \ - expected-warning {{defining a type within '__builtin_offsetof' is a Clang extension}} + expected-warning {{defining a type within '__builtin_offsetof' is a C23 extension}} { int a; int b; @@ -47,7 +47,7 @@ int struct_in_second_param(void) { int macro(void) { return offsetof(struct A // cpp-error {{'A' cannot be defined in a type specifier}} \ - expected-warning 2 {{defining a type within 'offsetof' is a Clang extension}} + expected-warning 2 {{defining a type within 'offsetof' is a C23 extension}} { int a; struct B // verifier seems to think the error is emitted by the macro diff --git a/clang/test/C/drs/dr4xx.c b/clang/test/C/drs/dr4xx.c index 30145dcfeef1..83d7b94cd679 100644 --- a/clang/test/C/drs/dr4xx.c +++ b/clang/test/C/drs/dr4xx.c @@ -1,7 +1,7 @@ -/* RUN: %clang_cc1 -std=c89 -verify=expected,c89only -pedantic -Wno-c11-extensions %s - RUN: %clang_cc1 -std=c99 -verify=expected -pedantic -Wno-c11-extensions %s - RUN: %clang_cc1 -std=c11 -verify=expected -pedantic %s - RUN: %clang_cc1 -std=c17 -verify=expected -pedantic %s +/* RUN: %clang_cc1 -std=c89 -verify=expected,c89only,pre-c23 -pedantic -Wno-c11-extensions %s + RUN: %clang_cc1 -std=c99 -verify=expected,pre-c23 -pedantic -Wno-c11-extensions %s + RUN: %clang_cc1 -std=c11 -verify=expected,pre-c23 -pedantic %s + RUN: %clang_cc1 -std=c17 -verify=expected,pre-c23 -pedantic %s RUN: %clang_cc1 -std=c2x -verify=expected -pedantic %s */ @@ -343,10 +343,13 @@ void dr496(void) { */ /* The DR asked a question about whether defining a new type within offsetof - * is allowed. C2x N2350 made this explicitly undefined behavior, but GCC and - * Clang both support it as an extension. + * is allowed. C23 N2350 had made this explicitly undefined behavior, but this + * was later overturned when C23 DE-137 was accepted, making it well-formed. + * + * Additionally, GCC and Clang both support it as an extension in pre-C23 + * mode. */ - (void)__builtin_offsetof(struct S { int a; }, a); /* expected-warning{{defining a type within '__builtin_offsetof' is a Clang extension}} */ + (void)__builtin_offsetof(struct S { int a; }, a); /* pre-c23-warning{{defining a type within '__builtin_offsetof' is a C23 extension}} */ } /* WG14 DR499: yes -- GitLab From 0dbd804a690720688d8234d8bdaee8f8f4fdcddc Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Fri, 29 Mar 2024 09:44:17 -0700 Subject: [PATCH 0009/1690] [scudo] Only init RingBuffer when needed. (#85994) Only attempt to initialize the ring buffer when tracking is enabled. Updated unit tests, and added a few new unit tests to verify the RingBuffer is not initialized by default. Verified that the two maps associated with the RingBuffer are not created in processes by default. --- compiler-rt/lib/scudo/standalone/combined.h | 31 +++++--- .../scudo/standalone/tests/combined_test.cpp | 78 ++++++++++++++++--- 2 files changed, 87 insertions(+), 22 deletions(-) diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h index f4dd90aac665..e7bc90cd0960 100644 --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -18,6 +18,7 @@ #include "local_cache.h" #include "mem_map.h" #include "memtag.h" +#include "mutex.h" #include "options.h" #include "quarantine.h" #include "report.h" @@ -178,17 +179,17 @@ public: Quarantine.init( static_cast(getFlags()->quarantine_size_kb << 10), static_cast(getFlags()->thread_local_quarantine_size_kb << 10)); - - mapAndInitializeRingBuffer(); } - void enableRingBuffer() { + void enableRingBuffer() NO_THREAD_SAFETY_ANALYSIS { AllocationRingBuffer *RB = getRingBuffer(); if (RB) RB->Depot->enable(); + RingBufferInitLock.unlock(); } - void disableRingBuffer() { + void disableRingBuffer() NO_THREAD_SAFETY_ANALYSIS { + RingBufferInitLock.lock(); AllocationRingBuffer *RB = getRingBuffer(); if (RB) RB->Depot->disable(); @@ -915,9 +916,11 @@ public: DCHECK(!Primary.Options.load().get(OptionBit::TrackAllocationStacks)); return; } - if (Track) + + if (Track) { + initRingBufferMaybe(); Primary.Options.set(OptionBit::TrackAllocationStacks); - else + } else Primary.Options.clear(OptionBit::TrackAllocationStacks); } @@ -1092,6 +1095,9 @@ private: 0, "invalid alignment"); + // Lock to initialize the RingBuffer + HybridMutex RingBufferInitLock; + // Pointer to memory mapped area starting with AllocationRingBuffer struct, // and immediately followed by Size elements of type Entry. atomic_uptr RingBufferAddress = {}; @@ -1546,11 +1552,16 @@ private: RBEntryStart)[N]; } - void mapAndInitializeRingBuffer() { - if (getFlags()->allocation_ring_buffer_size <= 0) + void initRingBufferMaybe() { + ScopedLock L(RingBufferInitLock); + if (getRingBuffer() != nullptr) return; - u32 AllocationRingBufferSize = - static_cast(getFlags()->allocation_ring_buffer_size); + + int ring_buffer_size = getFlags()->allocation_ring_buffer_size; + if (ring_buffer_size <= 0) + return; + + u32 AllocationRingBufferSize = static_cast(ring_buffer_size); // We store alloc and free stacks for each entry. constexpr u32 kStacksPerRingBufferEntry = 2; diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp index 6a311adc55e4..1a36155bcd42 100644 --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -867,32 +867,86 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateInPlaceStress) { } } +SCUDO_TYPED_TEST(ScudoCombinedTest, RingBufferDefaultDisabled) { + // The RingBuffer is not initialized until tracking is enabled for the + // first time. + auto *Allocator = this->Allocator.get(); + EXPECT_EQ(0u, Allocator->getRingBufferSize()); + EXPECT_EQ(nullptr, Allocator->getRingBufferAddress()); +} + +SCUDO_TYPED_TEST(ScudoCombinedTest, RingBufferInitOnce) { + auto *Allocator = this->Allocator.get(); + Allocator->setTrackAllocationStacks(true); + + auto RingBufferSize = Allocator->getRingBufferSize(); + ASSERT_GT(RingBufferSize, 0u); + auto *RingBufferAddress = Allocator->getRingBufferAddress(); + EXPECT_NE(nullptr, RingBufferAddress); + + // Enable tracking again to verify that the initialization only happens once. + Allocator->setTrackAllocationStacks(true); + ASSERT_EQ(RingBufferSize, Allocator->getRingBufferSize()); + EXPECT_EQ(RingBufferAddress, Allocator->getRingBufferAddress()); +} + SCUDO_TYPED_TEST(ScudoCombinedTest, RingBufferSize) { auto *Allocator = this->Allocator.get(); - auto Size = Allocator->getRingBufferSize(); - ASSERT_GT(Size, 0u); - EXPECT_EQ(Allocator->getRingBufferAddress()[Size - 1], '\0'); + Allocator->setTrackAllocationStacks(true); + + auto RingBufferSize = Allocator->getRingBufferSize(); + ASSERT_GT(RingBufferSize, 0u); + EXPECT_EQ(Allocator->getRingBufferAddress()[RingBufferSize - 1], '\0'); } SCUDO_TYPED_TEST(ScudoCombinedTest, RingBufferAddress) { auto *Allocator = this->Allocator.get(); - auto *Addr = Allocator->getRingBufferAddress(); - EXPECT_NE(Addr, nullptr); - EXPECT_EQ(Addr, Allocator->getRingBufferAddress()); + Allocator->setTrackAllocationStacks(true); + + auto *RingBufferAddress = Allocator->getRingBufferAddress(); + EXPECT_NE(RingBufferAddress, nullptr); + EXPECT_EQ(RingBufferAddress, Allocator->getRingBufferAddress()); +} + +SCUDO_TYPED_TEST(ScudoCombinedTest, StackDepotDefaultDisabled) { + // The StackDepot is not initialized until tracking is enabled for the + // first time. + auto *Allocator = this->Allocator.get(); + EXPECT_EQ(0u, Allocator->getStackDepotSize()); + EXPECT_EQ(nullptr, Allocator->getStackDepotAddress()); +} + +SCUDO_TYPED_TEST(ScudoCombinedTest, StackDepotInitOnce) { + auto *Allocator = this->Allocator.get(); + Allocator->setTrackAllocationStacks(true); + + auto StackDepotSize = Allocator->getStackDepotSize(); + EXPECT_GT(StackDepotSize, 0u); + auto *StackDepotAddress = Allocator->getStackDepotAddress(); + EXPECT_NE(nullptr, StackDepotAddress); + + // Enable tracking again to verify that the initialization only happens once. + Allocator->setTrackAllocationStacks(true); + EXPECT_EQ(StackDepotSize, Allocator->getStackDepotSize()); + EXPECT_EQ(StackDepotAddress, Allocator->getStackDepotAddress()); } SCUDO_TYPED_TEST(ScudoCombinedTest, StackDepotSize) { auto *Allocator = this->Allocator.get(); - auto Size = Allocator->getStackDepotSize(); - ASSERT_GT(Size, 0u); - EXPECT_EQ(Allocator->getStackDepotAddress()[Size - 1], '\0'); + Allocator->setTrackAllocationStacks(true); + + auto StackDepotSize = Allocator->getStackDepotSize(); + EXPECT_GT(StackDepotSize, 0u); + EXPECT_EQ(Allocator->getStackDepotAddress()[StackDepotSize - 1], '\0'); } SCUDO_TYPED_TEST(ScudoCombinedTest, StackDepotAddress) { auto *Allocator = this->Allocator.get(); - auto *Addr = Allocator->getStackDepotAddress(); - EXPECT_NE(Addr, nullptr); - EXPECT_EQ(Addr, Allocator->getStackDepotAddress()); + Allocator->setTrackAllocationStacks(true); + + auto *StackDepotAddress = Allocator->getStackDepotAddress(); + EXPECT_NE(StackDepotAddress, nullptr); + EXPECT_EQ(StackDepotAddress, Allocator->getStackDepotAddress()); } SCUDO_TYPED_TEST(ScudoCombinedTest, StackDepot) { -- GitLab From 60deb8b39afe9be90e30aa18d77ad129dacd4d55 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Fri, 29 Mar 2024 09:59:34 -0700 Subject: [PATCH 0010/1690] [InstallAPI][test] Tweak test to run on older CI config --- clang/test/InstallAPI/diagnostics-dsym.test | 26 +++++++++------------ 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/clang/test/InstallAPI/diagnostics-dsym.test b/clang/test/InstallAPI/diagnostics-dsym.test index ee2c8b32df29..45c69c09d286 100644 --- a/clang/test/InstallAPI/diagnostics-dsym.test +++ b/clang/test/InstallAPI/diagnostics-dsym.test @@ -4,20 +4,20 @@ ; RUN: split-file %s %t // Build a simple dylib with debug info. -; RUN: %clang --target=arm64-apple-macos13 -g -dynamiclib %t/foo.c \ +; RUN: %clang --target=x86_64-apple-macos10.15 -g -dynamiclib %t/foo.c \ ; RUN: -current_version 1 -compatibility_version 1 -L%t/usr/lib \ ; RUN: -save-temps \ ; RUN: -o %t/foo.dylib -install_name %t/foo.dylib ; RUN: dsymutil %t/foo.dylib -o %t/foo.dSYM -; RUN: not clang-installapi -x c++ --target=arm64-apple-macos13 \ +; RUN: not clang-installapi -x c++ --target=x86_64-apple-macos10.15 \ ; RUN: -install_name %t/foo.dylib \ ; RUN: -current_version 1 -compatibility_version 1 \ ; RUN: -o %t/output.tbd \ ; RUN: --verify-against=%t/foo.dylib --dsym=%t/foo.dSYM \ ; RUN: --verify-mode=Pedantic 2>&1 | FileCheck %s -; CHECK: violations found for arm64 +; CHECK: violations found for x86_64 ; CHECK: foo.c:5:0: error: no declaration found for exported symbol 'bar' in dynamic library ; CHECK: foo.c:1:0: error: no declaration found for exported symbol 'foo' in dynamic library @@ -29,15 +29,11 @@ extern char bar; char bar = 'a'; ;--- usr/lib/libSystem.tbd -{ - "main_library": { - "install_names": [ - {"name": "/usr/lib/libSystem.B.dylib"} - ], - "target_info": [ - {"target": "arm64-macos"} - ] - }, - "tapi_tbd_version": 5 -} - +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos ] +install-name: '/usr/lib/libSystem.B.dylib' +exports: + - targets: [ x86_64-macos ] + symbols: [ dyld_stub_binder ] +... -- GitLab From fdef5c4f453f0e0de02281a0a9b986667bc5010e Mon Sep 17 00:00:00 2001 From: Om Prakaash Date: Fri, 29 Mar 2024 10:15:22 -0700 Subject: [PATCH 0011/1690] Add bit width length modifier to printf (#82461) Resolves #81685. This adds support for wN and wfN length modifiers in fprintf. --- libc/docs/dev/printf_behavior.rst | 4 + libc/src/stdio/printf_core/converter_utils.h | 16 +++- libc/src/stdio/printf_core/core_structs.h | 9 +- libc/src/stdio/printf_core/int_converter.h | 5 +- libc/src/stdio/printf_core/parser.h | 69 ++++++++++++--- .../stdio/printf_core/write_int_converter.h | 2 + libc/test/UnitTest/PrintfMatcher.cpp | 6 ++ .../src/stdio/printf_core/parser_test.cpp | 36 ++++++++ libc/test/src/stdio/sprintf_test.cpp | 86 +++++++++++++++++++ 9 files changed, 215 insertions(+), 18 deletions(-) diff --git a/libc/docs/dev/printf_behavior.rst b/libc/docs/dev/printf_behavior.rst index 9548bfda57aa..c8b8ad45e987 100644 --- a/libc/docs/dev/printf_behavior.rst +++ b/libc/docs/dev/printf_behavior.rst @@ -173,6 +173,10 @@ If a number passed as a min width or precision value is out of range for an int, then it will be treated as the largest or smallest value in the int range (e.g. "%-999999999999.999999999999s" is the same as "%-2147483648.2147483647s"). +If a number passed as a bit width is less than or equal to zero, the conversion +is considered invalid. If the provided bit width is larger than the width of +uintmax_t, it will be clamped to the width of uintmax_t. + ---------- Conversion ---------- diff --git a/libc/src/stdio/printf_core/converter_utils.h b/libc/src/stdio/printf_core/converter_utils.h index 948fe816e9b7..a0e96a11be5b 100644 --- a/libc/src/stdio/printf_core/converter_utils.h +++ b/libc/src/stdio/printf_core/converter_utils.h @@ -18,7 +18,9 @@ namespace LIBC_NAMESPACE { namespace printf_core { -LIBC_INLINE uintmax_t apply_length_modifier(uintmax_t num, LengthModifier lm) { +LIBC_INLINE uintmax_t apply_length_modifier(uintmax_t num, + LengthSpec length_spec) { + auto [lm, bw] = length_spec; switch (lm) { case LengthModifier::none: return num & cpp::numeric_limits::max(); @@ -40,6 +42,18 @@ LIBC_INLINE uintmax_t apply_length_modifier(uintmax_t num, LengthModifier lm) { return num & cpp::numeric_limits::max(); case LengthModifier::j: return num; // j is intmax, so no mask is necessary. + case LengthModifier::w: + case LengthModifier::wf: { + uintmax_t mask; + if (bw == 0) { + mask = 0; + } else if (bw < sizeof(uintmax_t) * CHAR_BIT) { + mask = (static_cast(1) << bw) - 1; + } else { + mask = UINTMAX_MAX; + } + return num & mask; + } } __builtin_unreachable(); } diff --git a/libc/src/stdio/printf_core/core_structs.h b/libc/src/stdio/printf_core/core_structs.h index d3718b49d1b1..1e78f195a75e 100644 --- a/libc/src/stdio/printf_core/core_structs.h +++ b/libc/src/stdio/printf_core/core_structs.h @@ -22,7 +22,12 @@ namespace printf_core { // These length modifiers match the length modifiers in the format string, which // is why they are formatted differently from the rest of the file. -enum class LengthModifier { hh, h, l, ll, j, z, t, L, none }; +enum class LengthModifier { hh, h, l, ll, j, z, t, L, w, wf, none }; + +struct LengthSpec { + LengthModifier lm; + size_t bit_width; +}; enum FormatFlags : uint8_t { LEFT_JUSTIFIED = 0x01, // - @@ -44,6 +49,7 @@ struct FormatSection { // Format Specifier Values FormatFlags flags = FormatFlags(0); LengthModifier length_modifier = LengthModifier::none; + size_t bit_width = 0; int min_width = 0; int precision = -1; @@ -66,6 +72,7 @@ struct FormatSection { if (!((static_cast(flags) == static_cast(other.flags)) && (min_width == other.min_width) && (precision == other.precision) && + (bit_width == other.bit_width) && (length_modifier == other.length_modifier) && (conv_name == other.conv_name))) return false; diff --git a/libc/src/stdio/printf_core/int_converter.h b/libc/src/stdio/printf_core/int_converter.h index 2efbf53d4093..496e7bd1a56d 100644 --- a/libc/src/stdio/printf_core/int_converter.h +++ b/libc/src/stdio/printf_core/int_converter.h @@ -71,7 +71,6 @@ LIBC_INLINE int convert_int(Writer *writer, const FormatSection &to_conv) { uintmax_t num = static_cast(to_conv.conv_val_raw); bool is_negative = false; FormatFlags flags = to_conv.flags; - const char a = is_lower(to_conv.conv_name) ? 'a' : 'A'; // If the conversion is signed, then handle negative values. @@ -89,8 +88,8 @@ LIBC_INLINE int convert_int(Writer *writer, const FormatSection &to_conv) { ~(FormatFlags::FORCE_SIGN | FormatFlags::SPACE_PREFIX)); } - num = apply_length_modifier(num, to_conv.length_modifier); - + num = + apply_length_modifier(num, {to_conv.length_modifier, to_conv.bit_width}); cpp::array buf; auto str = details::num_to_strview(num, buf, to_conv.conv_name); if (!str) diff --git a/libc/src/stdio/printf_core/parser.h b/libc/src/stdio/printf_core/parser.h index 0876116a0bac..8e8c77e219fa 100644 --- a/libc/src/stdio/printf_core/parser.h +++ b/libc/src/stdio/printf_core/parser.h @@ -150,10 +150,10 @@ public: } } - LengthModifier lm = parse_length_modifier(&cur_pos); - + auto [lm, bw] = parse_length_modifier(&cur_pos); section.length_modifier = lm; section.conv_name = str[cur_pos]; + section.bit_width = bw; switch (str[cur_pos]) { case ('%'): // Regardless of options, a % conversion is always safe. The standard @@ -202,6 +202,21 @@ public: WRITE_ARG_VAL_SIMPLEST(section.conv_val_raw, ptrdiff_t, conv_index); break; + + case (LengthModifier::w): + case (LengthModifier::wf): + if (bw == 0) { + section.has_conv = false; + } else if (bw <= INT_WIDTH) { + WRITE_ARG_VAL_SIMPLEST(section.conv_val_raw, int, conv_index); + } else if (bw <= LONG_WIDTH) { + WRITE_ARG_VAL_SIMPLEST(section.conv_val_raw, long, conv_index); + } else if (bw <= LLONG_WIDTH) { + WRITE_ARG_VAL_SIMPLEST(section.conv_val_raw, long long, conv_index); + } else { + WRITE_ARG_VAL_SIMPLEST(section.conv_val_raw, intmax_t, conv_index); + } + break; } break; #ifndef LIBC_COPT_PRINTF_DISABLE_FLOAT @@ -306,38 +321,54 @@ private: // assumes that str[*local_pos] is inside a format specifier. It returns a // LengthModifier with the length modifier it found. It will advance local_pos // after the format specifier if one is found. - LIBC_INLINE LengthModifier parse_length_modifier(size_t *local_pos) { + LIBC_INLINE LengthSpec parse_length_modifier(size_t *local_pos) { switch (str[*local_pos]) { case ('l'): if (str[*local_pos + 1] == 'l') { *local_pos += 2; - return LengthModifier::ll; + return {LengthModifier::ll, 0}; + } else { + ++*local_pos; + return {LengthModifier::l, 0}; + } + case ('w'): { + LengthModifier lm; + if (str[*local_pos + 1] == 'f') { + *local_pos += 2; + lm = LengthModifier::wf; } else { ++*local_pos; - return LengthModifier::l; + lm = LengthModifier::w; } + if (internal::isdigit(str[*local_pos])) { + const auto result = internal::strtointeger(str + *local_pos, 10); + *local_pos += result.parsed_len; + return {lm, static_cast(cpp::max(0, result.value))}; + } + return {lm, 0}; + } case ('h'): if (str[*local_pos + 1] == 'h') { *local_pos += 2; - return LengthModifier::hh; + return {LengthModifier::hh, 0}; } else { ++*local_pos; - return LengthModifier::h; + return {LengthModifier::h, 0}; } case ('L'): ++*local_pos; - return LengthModifier::L; + return {LengthModifier::L, 0}; case ('j'): ++*local_pos; - return LengthModifier::j; + return {LengthModifier::j, 0}; case ('z'): ++*local_pos; - return LengthModifier::z; + return {LengthModifier::z, 0}; case ('t'): ++*local_pos; - return LengthModifier::t; + return {LengthModifier::t, 0}; default: - return LengthModifier::none; + return {LengthModifier::none, 0}; } } @@ -509,7 +540,7 @@ private: } } - LengthModifier lm = parse_length_modifier(&local_pos); + auto [lm, bw] = parse_length_modifier(&local_pos); // if we don't have an index for this conversion, then its position is // unknown and all this information is irrelevant. The rest of this @@ -560,6 +591,18 @@ private: case (LengthModifier::t): conv_size = type_desc_from_type(); break; + case (LengthModifier::w): + case (LengthModifier::wf): + if (bw <= INT_WIDTH) { + conv_size = type_desc_from_type(); + } else if (bw <= LONG_WIDTH) { + conv_size = type_desc_from_type(); + } else if (bw <= LLONG_WIDTH) { + conv_size = type_desc_from_type(); + } else { + conv_size = type_desc_from_type(); + } + break; } break; #ifndef LIBC_COPT_PRINTF_DISABLE_FLOAT diff --git a/libc/src/stdio/printf_core/write_int_converter.h b/libc/src/stdio/printf_core/write_int_converter.h index 0310905f36f1..18aa5c79897e 100644 --- a/libc/src/stdio/printf_core/write_int_converter.h +++ b/libc/src/stdio/printf_core/write_int_converter.h @@ -55,6 +55,8 @@ LIBC_INLINE int convert_write_int(Writer *writer, *reinterpret_cast(to_conv.conv_val_ptr) = written; break; case LengthModifier::j: + case LengthModifier::w: + case LengthModifier::wf: *reinterpret_cast(to_conv.conv_val_ptr) = written; break; } diff --git a/libc/test/UnitTest/PrintfMatcher.cpp b/libc/test/UnitTest/PrintfMatcher.cpp index 32f3be73307e..c8303815c922 100644 --- a/libc/test/UnitTest/PrintfMatcher.cpp +++ b/libc/test/UnitTest/PrintfMatcher.cpp @@ -39,6 +39,10 @@ namespace { case (LengthModifier::lm): \ tlog << #lm; \ break +#define CASE_LM_BIT_WIDTH(lm, bw) \ + case (LengthModifier::lm): \ + tlog << #lm << "\n\tbit width: :" << bw; \ + break static void display(FormatSection form) { tlog << "Raw String (len " << form.raw_string.size() << "): \""; @@ -67,6 +71,8 @@ static void display(FormatSection form) { CASE_LM(z); CASE_LM(t); CASE_LM(L); + CASE_LM_BIT_WIDTH(w, form.bit_width); + CASE_LM_BIT_WIDTH(wf, form.bit_width); } tlog << "\n"; tlog << "\tconversion name: " << form.conv_name << "\n"; diff --git a/libc/test/src/stdio/printf_core/parser_test.cpp b/libc/test/src/stdio/printf_core/parser_test.cpp index 0134277c4a1b..66d6dd0a86c4 100644 --- a/libc/test/src/stdio/printf_core/parser_test.cpp +++ b/libc/test/src/stdio/printf_core/parser_test.cpp @@ -223,6 +223,42 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArgWithLongLengthModifier) { ASSERT_PFORMAT_EQ(expected, format_arr[0]); } +TEST(LlvmLibcPrintfParserTest, EvalOneArgWithBitWidthLengthModifier) { + LIBC_NAMESPACE::printf_core::FormatSection format_arr[10]; + const char *str = "%w32d"; + long long arg1 = 12345; + evaluate(format_arr, str, arg1); + + LIBC_NAMESPACE::printf_core::FormatSection expected; + expected.has_conv = true; + + expected.raw_string = {str, 5}; + expected.length_modifier = LIBC_NAMESPACE::printf_core::LengthModifier::w; + expected.bit_width = 32; + expected.conv_val_raw = arg1; + expected.conv_name = 'd'; + + ASSERT_PFORMAT_EQ(expected, format_arr[0]); +} + +TEST(LlvmLibcPrintfParserTest, EvalOneArgWithFastBitWidthLengthModifier) { + LIBC_NAMESPACE::printf_core::FormatSection format_arr[10]; + const char *str = "%wf32d"; + long long arg1 = 12345; + evaluate(format_arr, str, arg1); + + LIBC_NAMESPACE::printf_core::FormatSection expected; + expected.has_conv = true; + + expected.raw_string = {str, 6}; + expected.length_modifier = LIBC_NAMESPACE::printf_core::LengthModifier::wf; + expected.bit_width = 32; + expected.conv_val_raw = arg1; + expected.conv_name = 'd'; + + ASSERT_PFORMAT_EQ(expected, format_arr[0]); +} + TEST(LlvmLibcPrintfParserTest, EvalOneArgWithAllOptions) { LIBC_NAMESPACE::printf_core::FormatSection format_arr[10]; const char *str = "% -056.78jd"; diff --git a/libc/test/src/stdio/sprintf_test.cpp b/libc/test/src/stdio/sprintf_test.cpp index 8dde95d02a96..ba31034146fe 100644 --- a/libc/test/src/stdio/sprintf_test.cpp +++ b/libc/test/src/stdio/sprintf_test.cpp @@ -169,6 +169,92 @@ TEST(LlvmLibcSPrintfTest, IntConv) { EXPECT_EQ(written, 20); ASSERT_STREQ(buff, "-9223372036854775808"); // ll min + written = LIBC_NAMESPACE::sprintf(buff, "%w3d", 5807); + EXPECT_EQ(written, 1); + ASSERT_STREQ(buff, "7"); + + written = LIBC_NAMESPACE::sprintf(buff, "%w3d", 1); + EXPECT_EQ(written, 1); + ASSERT_STREQ(buff, "1"); + + written = LIBC_NAMESPACE::sprintf(buff, "%w64d", 9223372036854775807ll); + EXPECT_EQ(written, 19); + ASSERT_STREQ(buff, "9223372036854775807"); + + written = LIBC_NAMESPACE::sprintf(buff, "%w-1d", 5807); + EXPECT_EQ(written, 5); + ASSERT_STREQ(buff, "%w-1d"); + + written = LIBC_NAMESPACE::sprintf(buff, "%w0d", 5807); + EXPECT_EQ(written, 4); + ASSERT_STREQ(buff, "%w0d"); + + written = LIBC_NAMESPACE::sprintf(buff, "%w999d", 9223372036854775807ll); + EXPECT_EQ(written, 19); + ASSERT_STREQ(buff, "9223372036854775807"); + + written = LIBC_NAMESPACE::sprintf(buff, "%winvalid%w1d", 5807, 5807); + EXPECT_EQ(written, 10); + ASSERT_STREQ(buff, "%winvalid1"); + + written = LIBC_NAMESPACE::sprintf(buff, "%w-1d%w1d", 5807, 5807); + EXPECT_EQ(written, 6); + ASSERT_STREQ(buff, "%w-1d1"); + + char format[64]; + char uintmax[128]; + LIBC_NAMESPACE::sprintf(format, "%%w%du", UINTMAX_WIDTH); + const int uintmax_len = LIBC_NAMESPACE::sprintf(uintmax, "%ju", UINTMAX_MAX); + written = LIBC_NAMESPACE::sprintf(buff, format, UINTMAX_MAX); + EXPECT_EQ(written, uintmax_len); + ASSERT_STREQ(buff, uintmax); + + written = LIBC_NAMESPACE::sprintf(buff, "%w64u", 18446744073709551615ull); + EXPECT_EQ(written, 20); + ASSERT_STREQ(buff, "18446744073709551615"); // ull max + + written = + LIBC_NAMESPACE::sprintf(buff, "%w64d", -9223372036854775807ll - 1ll); + EXPECT_EQ(written, 20); + ASSERT_STREQ(buff, "-9223372036854775808"); // ll min + + written = LIBC_NAMESPACE::sprintf(buff, "%wf3d", 5807); + EXPECT_EQ(written, 1); + ASSERT_STREQ(buff, "7"); + + written = LIBC_NAMESPACE::sprintf(buff, "%wf3d", 1); + EXPECT_EQ(written, 1); + ASSERT_STREQ(buff, "1"); + + written = LIBC_NAMESPACE::sprintf(buff, "%wf64u", 18446744073709551615ull); + EXPECT_EQ(written, 20); + ASSERT_STREQ(buff, "18446744073709551615"); // ull max + + written = + LIBC_NAMESPACE::sprintf(buff, "%wf64d", -9223372036854775807ll - 1ll); + EXPECT_EQ(written, 20); + ASSERT_STREQ(buff, "-9223372036854775808"); // ll min + + written = LIBC_NAMESPACE::sprintf(buff, "%wf0d", 5807); + EXPECT_EQ(written, 5); + ASSERT_STREQ(buff, "%wf0d"); + + written = LIBC_NAMESPACE::sprintf(buff, "%wf-1d", 5807); + EXPECT_EQ(written, 6); + ASSERT_STREQ(buff, "%wf-1d"); + + written = LIBC_NAMESPACE::sprintf(buff, "%wfinvalid%wf1d", 5807, 5807); + EXPECT_EQ(written, 11); + ASSERT_STREQ(buff, "%wfinvalid1"); + + written = LIBC_NAMESPACE::sprintf(buff, "%wf-1d%wf1d", 5807, 5807); + EXPECT_EQ(written, 7); + ASSERT_STREQ(buff, "%wf-1d1"); + + written = LIBC_NAMESPACE::sprintf(buff, "%wf999d", 9223372036854775807ll); + EXPECT_EQ(written, 19); + ASSERT_STREQ(buff, "9223372036854775807"); + // Min Width Tests. written = LIBC_NAMESPACE::sprintf(buff, "%4d", 789); -- GitLab From e74332a266e5f81411fb333bde56bf471f0a7ba6 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Fri, 29 Mar 2024 13:19:28 -0400 Subject: [PATCH 0012/1690] [HLSL][DXIL] HLSL's `round` should follow `roundeven` behavior (#87078) fixes #86999 --- clang/lib/Headers/hlsl/hlsl_intrinsics.h | 16 +++---- clang/lib/Sema/SemaChecking.cpp | 1 + clang/test/CodeGenHLSL/builtins/round.hlsl | 48 +++++++++---------- .../BuiltIns/half-float-only-errors.hlsl | 1 + llvm/lib/Target/DirectX/DXIL.td | 2 +- llvm/test/CodeGen/DirectX/round.ll | 35 +++++--------- llvm/test/CodeGen/DirectX/round_error.ll | 4 +- 7 files changed, 50 insertions(+), 57 deletions(-) diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h index a34e72402c0e..9fb6204f90c9 100644 --- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h +++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h @@ -1248,25 +1248,25 @@ float4 rsqrt(float4); /// rounded to the nearest even value. _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_round) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_roundeven) half round(half); _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_round) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_roundeven) half2 round(half2); _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_round) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_roundeven) half3 round(half3); _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_round) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_roundeven) half4 round(half4); -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_round) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_roundeven) float round(float); -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_round) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_roundeven) float2 round(float2); -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_round) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_roundeven) float3 round(float3); -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_round) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_roundeven) float4 round(float4); //===----------------------------------------------------------------------===// diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 2e4e18a3ebf7..2684535d8e53 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -5650,6 +5650,7 @@ bool Sema::CheckHLSLBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { case Builtin::BI__builtin_elementwise_log2: case Builtin::BI__builtin_elementwise_log10: case Builtin::BI__builtin_elementwise_pow: + case Builtin::BI__builtin_elementwise_roundeven: case Builtin::BI__builtin_elementwise_sin: case Builtin::BI__builtin_elementwise_sqrt: case Builtin::BI__builtin_elementwise_trunc: { diff --git a/clang/test/CodeGenHLSL/builtins/round.hlsl b/clang/test/CodeGenHLSL/builtins/round.hlsl index b9f35bd3712d..33d761dbdfbe 100644 --- a/clang/test/CodeGenHLSL/builtins/round.hlsl +++ b/clang/test/CodeGenHLSL/builtins/round.hlsl @@ -7,47 +7,47 @@ // RUN: -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF // NATIVE_HALF: define noundef half @ -// NATIVE_HALF: %elt.round = call half @llvm.round.f16( -// NATIVE_HALF: ret half %elt.round +// NATIVE_HALF: %elt.roundeven = call half @llvm.roundeven.f16( +// NATIVE_HALF: ret half %elt.roundeven // NO_HALF: define noundef float @"?test_round_half@@YA$halff@$halff@@Z"( -// NO_HALF: %elt.round = call float @llvm.round.f32( -// NO_HALF: ret float %elt.round +// NO_HALF: %elt.roundeven = call float @llvm.roundeven.f32( +// NO_HALF: ret float %elt.roundeven half test_round_half(half p0) { return round(p0); } // NATIVE_HALF: define noundef <2 x half> @ -// NATIVE_HALF: %elt.round = call <2 x half> @llvm.round.v2f16 -// NATIVE_HALF: ret <2 x half> %elt.round +// NATIVE_HALF: %elt.roundeven = call <2 x half> @llvm.roundeven.v2f16 +// NATIVE_HALF: ret <2 x half> %elt.roundeven // NO_HALF: define noundef <2 x float> @ -// NO_HALF: %elt.round = call <2 x float> @llvm.round.v2f32( -// NO_HALF: ret <2 x float> %elt.round +// NO_HALF: %elt.roundeven = call <2 x float> @llvm.roundeven.v2f32( +// NO_HALF: ret <2 x float> %elt.roundeven half2 test_round_half2(half2 p0) { return round(p0); } // NATIVE_HALF: define noundef <3 x half> @ -// NATIVE_HALF: %elt.round = call <3 x half> @llvm.round.v3f16 -// NATIVE_HALF: ret <3 x half> %elt.round +// NATIVE_HALF: %elt.roundeven = call <3 x half> @llvm.roundeven.v3f16 +// NATIVE_HALF: ret <3 x half> %elt.roundeven // NO_HALF: define noundef <3 x float> @ -// NO_HALF: %elt.round = call <3 x float> @llvm.round.v3f32( -// NO_HALF: ret <3 x float> %elt.round +// NO_HALF: %elt.roundeven = call <3 x float> @llvm.roundeven.v3f32( +// NO_HALF: ret <3 x float> %elt.roundeven half3 test_round_half3(half3 p0) { return round(p0); } // NATIVE_HALF: define noundef <4 x half> @ -// NATIVE_HALF: %elt.round = call <4 x half> @llvm.round.v4f16 -// NATIVE_HALF: ret <4 x half> %elt.round +// NATIVE_HALF: %elt.roundeven = call <4 x half> @llvm.roundeven.v4f16 +// NATIVE_HALF: ret <4 x half> %elt.roundeven // NO_HALF: define noundef <4 x float> @ -// NO_HALF: %elt.round = call <4 x float> @llvm.round.v4f32( -// NO_HALF: ret <4 x float> %elt.round +// NO_HALF: %elt.roundeven = call <4 x float> @llvm.roundeven.v4f32( +// NO_HALF: ret <4 x float> %elt.roundeven half4 test_round_half4(half4 p0) { return round(p0); } // CHECK: define noundef float @ -// CHECK: %elt.round = call float @llvm.round.f32( -// CHECK: ret float %elt.round +// CHECK: %elt.roundeven = call float @llvm.roundeven.f32( +// CHECK: ret float %elt.roundeven float test_round_float(float p0) { return round(p0); } // CHECK: define noundef <2 x float> @ -// CHECK: %elt.round = call <2 x float> @llvm.round.v2f32 -// CHECK: ret <2 x float> %elt.round +// CHECK: %elt.roundeven = call <2 x float> @llvm.roundeven.v2f32 +// CHECK: ret <2 x float> %elt.roundeven float2 test_round_float2(float2 p0) { return round(p0); } // CHECK: define noundef <3 x float> @ -// CHECK: %elt.round = call <3 x float> @llvm.round.v3f32 -// CHECK: ret <3 x float> %elt.round +// CHECK: %elt.roundeven = call <3 x float> @llvm.roundeven.v3f32 +// CHECK: ret <3 x float> %elt.roundeven float3 test_round_float3(float3 p0) { return round(p0); } // CHECK: define noundef <4 x float> @ -// CHECK: %elt.round = call <4 x float> @llvm.round.v4f32 -// CHECK: ret <4 x float> %elt.round +// CHECK: %elt.roundeven = call <4 x float> @llvm.roundeven.v4f32 +// CHECK: ret <4 x float> %elt.roundeven float4 test_round_float4(float4 p0) { return round(p0); } diff --git a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl index c56986b7f862..98c02c38675f 100644 --- a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl @@ -8,6 +8,7 @@ // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_log10 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_sin // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_sqrt +// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_roundeven // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_trunc double2 test_double_builtin(double2 p0) { diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index c5d7ee76275f..a131bcc7dafe 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -285,7 +285,7 @@ def RSqrt : DXILOpMapping<25, unary, int_dx_rsqrt, "Returns the reciprocal of the square root of the specified value." "rsqrt(x) = 1 / sqrt(x).", [llvm_halforfloat_ty, LLVMMatchType<0>]>; -def Round : DXILOpMapping<26, unary, int_round, +def Round : DXILOpMapping<26, unary, int_roundeven, "Returns the input rounded to the nearest integer" "within a floating-point type.", [llvm_halforfloat_ty, LLVMMatchType<0>]>; diff --git a/llvm/test/CodeGen/DirectX/round.ll b/llvm/test/CodeGen/DirectX/round.ll index 5d53a794b763..e0a3772ebca8 100644 --- a/llvm/test/CodeGen/DirectX/round.ll +++ b/llvm/test/CodeGen/DirectX/round.ll @@ -1,31 +1,22 @@ ; RUN: opt -S -dxil-op-lower < %s | FileCheck %s ; Make sure dxil operation function calls for round are generated for float and half. -; CHECK:call float @dx.op.unary.f32(i32 26, float %{{.*}}) -; CHECK:call half @dx.op.unary.f16(i32 26, half %{{.*}}) -target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64" -target triple = "dxil-pc-shadermodel6.7-library" - -; Function Attrs: noinline nounwind optnone -define noundef float @round_float(float noundef %a) #0 { +; CHECK-LABEL: round_half +define noundef half @round_half(half noundef %a) { entry: - %a.addr = alloca float, align 4 - store float %a, ptr %a.addr, align 4 - %0 = load float, ptr %a.addr, align 4 - %elt.round = call float @llvm.round.f32(float %0) - ret float %elt.round +; CHECK: call half @dx.op.unary.f16(i32 26, half %{{.*}}) + %elt.roundeven = call half @llvm.roundeven.f16(half %a) + ret half %elt.roundeven } -; Function Attrs: nocallback nofree nosync nounwind readnone speculatable willreturn -declare float @llvm.round.f32(float) #1 - -; Function Attrs: noinline nounwind optnone -define noundef half @round_half(half noundef %a) #0 { +; CHECK-LABEL: round_float +define noundef float @round_float(float noundef %a) { entry: - %a.addr = alloca half, align 2 - store half %a, ptr %a.addr, align 2 - %0 = load half, ptr %a.addr, align 2 - %elt.round = call half @llvm.round.f16(half %0) - ret half %elt.round +; CHECK: call float @dx.op.unary.f32(i32 26, float %{{.*}}) + %elt.roundeven = call float @llvm.roundeven.f32(float %a) + ret float %elt.roundeven } + +declare half @llvm.roundeven.f16(half) +declare float @llvm.roundeven.f32(float) diff --git a/llvm/test/CodeGen/DirectX/round_error.ll b/llvm/test/CodeGen/DirectX/round_error.ll index 3bd87b2bbf02..2d27fbb5ee20 100644 --- a/llvm/test/CodeGen/DirectX/round_error.ll +++ b/llvm/test/CodeGen/DirectX/round_error.ll @@ -8,6 +8,6 @@ entry: %a.addr = alloca double, align 8 store double %a, ptr %a.addr, align 8 %0 = load double, ptr %a.addr, align 8 - %elt.round = call double @llvm.round.f64(double %0) - ret double %elt.round + %elt.roundeven = call double @llvm.roundeven.f64(double %0) + ret double %elt.roundeven } -- GitLab From 838b118d394138a746ffb4c113fb97b0f8904bd2 Mon Sep 17 00:00:00 2001 From: Michael Jones Date: Fri, 29 Mar 2024 10:32:33 -0700 Subject: [PATCH 0013/1690] [libc] Fix missing UINTMAX_WIDTH (#87092) In patch #82461 the sprintf tests were made to use UINTMAX_WIDTH which isn't defined on all systems. This patch changes it to sizeof(uintmax_t)*CHAR_BIT which is more portable. --- libc/test/src/stdio/sprintf_test.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libc/test/src/stdio/sprintf_test.cpp b/libc/test/src/stdio/sprintf_test.cpp index ba31034146fe..8e9870f71a95 100644 --- a/libc/test/src/stdio/sprintf_test.cpp +++ b/libc/test/src/stdio/sprintf_test.cpp @@ -203,9 +203,10 @@ TEST(LlvmLibcSPrintfTest, IntConv) { char format[64]; char uintmax[128]; - LIBC_NAMESPACE::sprintf(format, "%%w%du", UINTMAX_WIDTH); - const int uintmax_len = LIBC_NAMESPACE::sprintf(uintmax, "%ju", UINTMAX_MAX); - written = LIBC_NAMESPACE::sprintf(buff, format, UINTMAX_MAX); + LIBC_NAMESPACE::sprintf(format, "%%w%du", sizeof(uintmax_t) * CHAR_BIT); + const int uintmax_len = + LIBC_NAMESPACE::sprintf(uintmax, "%ju", sizeof(uintmax_t) * CHAR_BIT); + written = LIBC_NAMESPACE::sprintf(buff, format, sizeof(uintmax_t) * CHAR_BIT); EXPECT_EQ(written, uintmax_len); ASSERT_STREQ(buff, uintmax); -- GitLab From bdb60e6f0c8e89abf9bdf36411348db304ca65ba Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Fri, 29 Mar 2024 10:33:14 -0700 Subject: [PATCH 0014/1690] [InstallAPI][test] Add requires x86_64 for hardcoded target test --- clang/test/InstallAPI/diagnostics-dsym.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/InstallAPI/diagnostics-dsym.test b/clang/test/InstallAPI/diagnostics-dsym.test index 45c69c09d286..8a1b394f2f86 100644 --- a/clang/test/InstallAPI/diagnostics-dsym.test +++ b/clang/test/InstallAPI/diagnostics-dsym.test @@ -1,4 +1,4 @@ -; REQUIRES: system-darwin +; REQUIRES: 86_64-darwin ; RUN: rm -rf %t ; RUN: split-file %s %t -- GitLab From a1a8bb1d3ae9a535526aba9514e87f76e2d040fa Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 29 Mar 2024 12:49:16 -0500 Subject: [PATCH 0015/1690] [libc] Change RPC interface to not use device ids (#87087) Summary: The current implementation of RPC tied everything to device IDs and forced us to do init / shutdown to manage some global state. This turned out to be a bad idea in situations where we want to track multiple hetergeneous devices that may report the same device ID in the same process. This patch changes the interface to instead create an opaque handle to the internal device and simply allocates it via `new`. The user will then take this device and store it to interface with the attached device. This interface puts the burden of tracking the device identifier to mapped d evices onto the user, but in return heavily simplifies the implementation. --- libc/utils/gpu/loader/Loader.h | 8 +- libc/utils/gpu/loader/amdgpu/Loader.cpp | 47 ++++---- libc/utils/gpu/loader/nvptx/Loader.cpp | 39 +++--- libc/utils/gpu/server/llvmlibc_rpc_server.h | 29 ++--- libc/utils/gpu/server/rpc_server.cpp | 112 +++++------------- .../plugins-nextgen/common/include/RPC.h | 7 +- .../common/src/PluginInterface.cpp | 2 +- .../plugins-nextgen/common/src/RPC.cpp | 44 +++---- 8 files changed, 107 insertions(+), 181 deletions(-) diff --git a/libc/utils/gpu/loader/Loader.h b/libc/utils/gpu/loader/Loader.h index 933803837019..9c7d328930c2 100644 --- a/libc/utils/gpu/loader/Loader.h +++ b/libc/utils/gpu/loader/Loader.h @@ -108,11 +108,11 @@ inline void handle_error(rpc_status_t) { } template -inline void register_rpc_callbacks(uint32_t device_id) { +inline void register_rpc_callbacks(rpc_device_t device) { static_assert(lane_size == 32 || lane_size == 64, "Invalid Lane size"); // Register the ping test for the `libc` tests. rpc_register_callback( - device_id, static_cast(RPC_TEST_INCREMENT), + device, static_cast(RPC_TEST_INCREMENT), [](rpc_port_t port, void *data) { rpc_recv_and_send( port, @@ -125,7 +125,7 @@ inline void register_rpc_callbacks(uint32_t device_id) { // Register the interface test callbacks. rpc_register_callback( - device_id, static_cast(RPC_TEST_INTERFACE), + device, static_cast(RPC_TEST_INTERFACE), [](rpc_port_t port, void *data) { uint64_t cnt = 0; bool end_with_recv; @@ -207,7 +207,7 @@ inline void register_rpc_callbacks(uint32_t device_id) { // Register the stream test handler. rpc_register_callback( - device_id, static_cast(RPC_TEST_STREAM), + device, static_cast(RPC_TEST_STREAM), [](rpc_port_t port, void *data) { uint64_t sizes[lane_size] = {0}; void *dst[lane_size] = {nullptr}; diff --git a/libc/utils/gpu/loader/amdgpu/Loader.cpp b/libc/utils/gpu/loader/amdgpu/Loader.cpp index e3911eda2bd8..35840c6910bd 100644 --- a/libc/utils/gpu/loader/amdgpu/Loader.cpp +++ b/libc/utils/gpu/loader/amdgpu/Loader.cpp @@ -153,7 +153,8 @@ template hsa_status_t launch_kernel(hsa_agent_t dev_agent, hsa_executable_t executable, hsa_amd_memory_pool_t kernargs_pool, hsa_amd_memory_pool_t coarsegrained_pool, - hsa_queue_t *queue, const LaunchParameters ¶ms, + hsa_queue_t *queue, rpc_device_t device, + const LaunchParameters ¶ms, const char *kernel_name, args_t kernel_args) { // Look up the '_start' kernel in the loaded executable. hsa_executable_symbol_t symbol; @@ -162,10 +163,9 @@ hsa_status_t launch_kernel(hsa_agent_t dev_agent, hsa_executable_t executable, return err; // Register RPC callbacks for the malloc and free functions on HSA. - uint32_t device_id = 0; auto tuple = std::make_tuple(dev_agent, coarsegrained_pool); rpc_register_callback( - device_id, RPC_MALLOC, + device, RPC_MALLOC, [](rpc_port_t port, void *data) { auto malloc_handler = [](rpc_buffer_t *buffer, void *data) -> void { auto &[dev_agent, pool] = *static_cast(data); @@ -182,7 +182,7 @@ hsa_status_t launch_kernel(hsa_agent_t dev_agent, hsa_executable_t executable, }, &tuple); rpc_register_callback( - device_id, RPC_FREE, + device, RPC_FREE, [](rpc_port_t port, void *data) { auto free_handler = [](rpc_buffer_t *buffer, void *) { if (hsa_status_t err = hsa_amd_memory_pool_free( @@ -284,12 +284,12 @@ hsa_status_t launch_kernel(hsa_agent_t dev_agent, hsa_executable_t executable, while (hsa_signal_wait_scacquire( packet->completion_signal, HSA_SIGNAL_CONDITION_EQ, 0, /*timeout_hint=*/1024, HSA_WAIT_STATE_ACTIVE) != 0) - if (rpc_status_t err = rpc_handle_server(device_id)) + if (rpc_status_t err = rpc_handle_server(device)) handle_error(err); // Handle the server one more time in case the kernel exited with a pending // send still in flight. - if (rpc_status_t err = rpc_handle_server(device_id)) + if (rpc_status_t err = rpc_handle_server(device)) handle_error(err); // Destroy the resources acquired to launch the kernel and return. @@ -342,8 +342,6 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, handle_error(err); // Obtain a single agent for the device and host to use the HSA memory model. - uint32_t num_devices = 1; - uint32_t device_id = 0; hsa_agent_t dev_agent; hsa_agent_t host_agent; if (hsa_status_t err = get_agent(&dev_agent)) @@ -433,8 +431,6 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, handle_error(err); // Set up the RPC server. - if (rpc_status_t err = rpc_init(num_devices)) - handle_error(err); auto tuple = std::make_tuple(dev_agent, finegrained_pool); auto rpc_alloc = [](uint64_t size, void *data) { auto &[dev_agent, finegrained_pool] = *static_cast(data); @@ -445,15 +441,16 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, hsa_amd_agents_allow_access(1, &dev_agent, nullptr, dev_ptr); return dev_ptr; }; - if (rpc_status_t err = rpc_server_init(device_id, RPC_MAXIMUM_PORT_COUNT, + rpc_device_t device; + if (rpc_status_t err = rpc_server_init(&device, RPC_MAXIMUM_PORT_COUNT, wavefront_size, rpc_alloc, &tuple)) handle_error(err); // Register callbacks for the RPC unit tests. if (wavefront_size == 32) - register_rpc_callbacks<32>(device_id); + register_rpc_callbacks<32>(device); else if (wavefront_size == 64) - register_rpc_callbacks<64>(device_id); + register_rpc_callbacks<64>(device); else handle_error("Invalid wavefront size"); @@ -483,10 +480,10 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, handle_error(err); void *rpc_client_buffer; - if (hsa_status_t err = hsa_amd_memory_lock( - const_cast(rpc_get_client_buffer(device_id)), - rpc_get_client_size(), - /*agents=*/nullptr, 0, &rpc_client_buffer)) + if (hsa_status_t err = + hsa_amd_memory_lock(const_cast(rpc_get_client_buffer(device)), + rpc_get_client_size(), + /*agents=*/nullptr, 0, &rpc_client_buffer)) handle_error(err); // Copy the RPC client buffer to the address pointed to by the symbol. @@ -496,7 +493,7 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, handle_error(err); if (hsa_status_t err = hsa_amd_memory_unlock( - const_cast(rpc_get_client_buffer(device_id)))) + const_cast(rpc_get_client_buffer(device)))) handle_error(err); if (hsa_status_t err = hsa_amd_memory_pool_free(rpc_client_host)) handle_error(err); @@ -549,13 +546,13 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, begin_args_t init_args = {argc, dev_argv, dev_envp}; if (hsa_status_t err = launch_kernel( dev_agent, executable, kernargs_pool, coarsegrained_pool, queue, - single_threaded_params, "_begin.kd", init_args)) + device, single_threaded_params, "_begin.kd", init_args)) handle_error(err); start_args_t args = {argc, dev_argv, dev_envp, dev_ret}; - if (hsa_status_t err = - launch_kernel(dev_agent, executable, kernargs_pool, - coarsegrained_pool, queue, params, "_start.kd", args)) + if (hsa_status_t err = launch_kernel(dev_agent, executable, kernargs_pool, + coarsegrained_pool, queue, device, + params, "_start.kd", args)) handle_error(err); void *host_ret; @@ -575,11 +572,11 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, end_args_t fini_args = {ret}; if (hsa_status_t err = launch_kernel( dev_agent, executable, kernargs_pool, coarsegrained_pool, queue, - single_threaded_params, "_end.kd", fini_args)) + device, single_threaded_params, "_end.kd", fini_args)) handle_error(err); if (rpc_status_t err = rpc_server_shutdown( - device_id, [](void *ptr, void *) { hsa_amd_memory_pool_free(ptr); }, + device, [](void *ptr, void *) { hsa_amd_memory_pool_free(ptr); }, nullptr)) handle_error(err); @@ -600,8 +597,6 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, if (hsa_status_t err = hsa_code_object_destroy(object)) handle_error(err); - if (rpc_status_t err = rpc_shutdown()) - handle_error(err); if (hsa_status_t err = hsa_shut_down()) handle_error(err); diff --git a/libc/utils/gpu/loader/nvptx/Loader.cpp b/libc/utils/gpu/loader/nvptx/Loader.cpp index 5388f287063b..1818932f0a96 100644 --- a/libc/utils/gpu/loader/nvptx/Loader.cpp +++ b/libc/utils/gpu/loader/nvptx/Loader.cpp @@ -154,8 +154,8 @@ Expected get_ctor_dtor_array(const void *image, const size_t size, template CUresult launch_kernel(CUmodule binary, CUstream stream, - const LaunchParameters ¶ms, const char *kernel_name, - args_t kernel_args) { + rpc_device_t rpc_device, const LaunchParameters ¶ms, + const char *kernel_name, args_t kernel_args) { // look up the '_start' kernel in the loaded module. CUfunction function; if (CUresult err = cuModuleGetFunction(&function, binary, kernel_name)) @@ -175,11 +175,10 @@ CUresult launch_kernel(CUmodule binary, CUstream stream, handle_error(err); // Register RPC callbacks for the malloc and free functions on HSA. - uint32_t device_id = 0; - register_rpc_callbacks<32>(device_id); + register_rpc_callbacks<32>(rpc_device); rpc_register_callback( - device_id, RPC_MALLOC, + rpc_device, RPC_MALLOC, [](rpc_port_t port, void *data) { auto malloc_handler = [](rpc_buffer_t *buffer, void *data) -> void { CUstream memory_stream = *static_cast(data); @@ -197,7 +196,7 @@ CUresult launch_kernel(CUmodule binary, CUstream stream, }, &memory_stream); rpc_register_callback( - device_id, RPC_FREE, + rpc_device, RPC_FREE, [](rpc_port_t port, void *data) { auto free_handler = [](rpc_buffer_t *buffer, void *data) { CUstream memory_stream = *static_cast(data); @@ -219,12 +218,12 @@ CUresult launch_kernel(CUmodule binary, CUstream stream, // Wait until the kernel has completed execution on the device. Periodically // check the RPC client for work to be performed on the server. while (cuStreamQuery(stream) == CUDA_ERROR_NOT_READY) - if (rpc_status_t err = rpc_handle_server(device_id)) + if (rpc_status_t err = rpc_handle_server(rpc_device)) handle_error(err); // Handle the server one more time in case the kernel exited with a pending // send still in flight. - if (rpc_status_t err = rpc_handle_server(device_id)) + if (rpc_status_t err = rpc_handle_server(rpc_device)) handle_error(err); return CUDA_SUCCESS; @@ -235,7 +234,6 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, if (CUresult err = cuInit(0)) handle_error(err); // Obtain the first device found on the system. - uint32_t num_devices = 1; uint32_t device_id = 0; CUdevice device; if (CUresult err = cuDeviceGet(&device, device_id)) @@ -294,9 +292,6 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, if (CUresult err = cuMemsetD32(dev_ret, 0, 1)) handle_error(err); - if (rpc_status_t err = rpc_init(num_devices)) - handle_error(err); - uint32_t warp_size = 32; auto rpc_alloc = [](uint64_t size, void *) -> void * { void *dev_ptr; @@ -304,7 +299,8 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, handle_error(err); return dev_ptr; }; - if (rpc_status_t err = rpc_server_init(device_id, RPC_MAXIMUM_PORT_COUNT, + rpc_device_t rpc_device; + if (rpc_status_t err = rpc_server_init(&rpc_device, RPC_MAXIMUM_PORT_COUNT, warp_size, rpc_alloc, nullptr)) handle_error(err); @@ -321,19 +317,20 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, cuMemcpyDtoH(&rpc_client_host, rpc_client_dev, sizeof(void *))) handle_error(err); if (CUresult err = - cuMemcpyHtoD(rpc_client_host, rpc_get_client_buffer(device_id), + cuMemcpyHtoD(rpc_client_host, rpc_get_client_buffer(rpc_device), rpc_get_client_size())) handle_error(err); LaunchParameters single_threaded_params = {1, 1, 1, 1, 1, 1}; begin_args_t init_args = {argc, dev_argv, dev_envp}; - if (CUresult err = launch_kernel(binary, stream, single_threaded_params, - "_begin", init_args)) + if (CUresult err = launch_kernel(binary, stream, rpc_device, + single_threaded_params, "_begin", init_args)) handle_error(err); start_args_t args = {argc, dev_argv, dev_envp, reinterpret_cast(dev_ret)}; - if (CUresult err = launch_kernel(binary, stream, params, "_start", args)) + if (CUresult err = + launch_kernel(binary, stream, rpc_device, params, "_start", args)) handle_error(err); // Copy the return value back from the kernel and wait. @@ -345,8 +342,8 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, handle_error(err); end_args_t fini_args = {host_ret}; - if (CUresult err = launch_kernel(binary, stream, single_threaded_params, - "_end", fini_args)) + if (CUresult err = launch_kernel(binary, stream, rpc_device, + single_threaded_params, "_end", fini_args)) handle_error(err); // Free the memory allocated for the device. @@ -357,7 +354,7 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, if (CUresult err = cuMemFreeHost(dev_argv)) handle_error(err); if (rpc_status_t err = rpc_server_shutdown( - device_id, [](void *ptr, void *) { cuMemFreeHost(ptr); }, nullptr)) + rpc_device, [](void *ptr, void *) { cuMemFreeHost(ptr); }, nullptr)) handle_error(err); // Destroy the context and the loaded binary. @@ -365,7 +362,5 @@ int load(int argc, char **argv, char **envp, void *image, size_t size, handle_error(err); if (CUresult err = cuDevicePrimaryCtxRelease(device)) handle_error(err); - if (rpc_status_t err = rpc_shutdown()) - handle_error(err); return host_ret; } diff --git a/libc/utils/gpu/server/llvmlibc_rpc_server.h b/libc/utils/gpu/server/llvmlibc_rpc_server.h index b7f2a463b1f5..b0cf2f916b38 100644 --- a/libc/utils/gpu/server/llvmlibc_rpc_server.h +++ b/libc/utils/gpu/server/llvmlibc_rpc_server.h @@ -27,10 +27,8 @@ typedef enum { RPC_STATUS_SUCCESS = 0x0, RPC_STATUS_CONTINUE = 0x1, RPC_STATUS_ERROR = 0x1000, - RPC_STATUS_OUT_OF_RANGE = 0x1001, - RPC_STATUS_UNHANDLED_OPCODE = 0x1002, - RPC_STATUS_INVALID_LANE_SIZE = 0x1003, - RPC_STATUS_NOT_INITIALIZED = 0x1004, + RPC_STATUS_UNHANDLED_OPCODE = 0x1001, + RPC_STATUS_INVALID_LANE_SIZE = 0x1002, } rpc_status_t; /// A struct containing an opaque handle to an RPC port. This is what allows the @@ -45,6 +43,11 @@ typedef struct rpc_buffer_s { uint64_t data[8]; } rpc_buffer_t; +/// An opaque handle to an RPC server that can be attached to a device. +typedef struct rpc_device_s { + uintptr_t handle; +} rpc_device_t; + /// A function used to allocate \p bytes for use by the RPC server and client. /// The memory should support asynchronous and atomic access from both the /// client and server. @@ -60,34 +63,28 @@ typedef void (*rpc_opcode_callback_ty)(rpc_port_t port, void *data); /// A callback function to use the port to receive or send a \p buffer. typedef void (*rpc_port_callback_ty)(rpc_buffer_t *buffer, void *data); -/// Initialize the rpc library for general use on \p num_devices. -rpc_status_t rpc_init(uint32_t num_devices); - -/// Shut down the rpc interface. -rpc_status_t rpc_shutdown(void); - -/// Initialize the server for a given device. -rpc_status_t rpc_server_init(uint32_t device_id, uint64_t num_ports, +/// Initialize the server for a given device and return it in \p device. +rpc_status_t rpc_server_init(rpc_device_t *rpc_device, uint64_t num_ports, uint32_t lane_size, rpc_alloc_ty alloc, void *data); /// Shut down the server for a given device. -rpc_status_t rpc_server_shutdown(uint32_t device_id, rpc_free_ty dealloc, +rpc_status_t rpc_server_shutdown(rpc_device_t rpc_device, rpc_free_ty dealloc, void *data); /// Queries the RPC clients at least once and performs server-side work if there /// are any active requests. Runs until all work on the server is completed. -rpc_status_t rpc_handle_server(uint32_t device_id); +rpc_status_t rpc_handle_server(rpc_device_t rpc_device); /// Register a callback to handle an opcode from the RPC client. The associated /// data must remain accessible as long as the user intends to handle the server /// with this callback. -rpc_status_t rpc_register_callback(uint32_t device_id, uint16_t opcode, +rpc_status_t rpc_register_callback(rpc_device_t rpc_device, uint16_t opcode, rpc_opcode_callback_ty callback, void *data); /// Obtain a pointer to a local client buffer that can be copied directly to the /// other process using the address stored at the rpc client symbol name. -const void *rpc_get_client_buffer(uint32_t device_id); +const void *rpc_get_client_buffer(rpc_device_t device); /// Returns the size of the client in bytes to be used for a memory copy. uint64_t rpc_get_client_size(); diff --git a/libc/utils/gpu/server/rpc_server.cpp b/libc/utils/gpu/server/rpc_server.cpp index 46ad98fa02cc..fd306642fdcc 100644 --- a/libc/utils/gpu/server/rpc_server.cpp +++ b/libc/utils/gpu/server/rpc_server.cpp @@ -248,127 +248,75 @@ struct Device { std::unordered_map callback_data; }; -// A struct containing all the runtime state required to run the RPC server. -struct State { - State(uint32_t num_devices) - : num_devices(num_devices), devices(num_devices), reference_count(0u) {} - uint32_t num_devices; - std::vector> devices; - std::atomic_uint32_t reference_count; -}; - -static std::mutex startup_mutex; - -static State *state; - -rpc_status_t rpc_init(uint32_t num_devices) { - std::scoped_lock lock(startup_mutex); - if (!state) - state = new State(num_devices); - - if (state->reference_count == std::numeric_limits::max()) - return RPC_STATUS_ERROR; - - state->reference_count++; - - return RPC_STATUS_SUCCESS; -} - -rpc_status_t rpc_shutdown(void) { - if (state && state->reference_count-- == 1) - delete state; - - return RPC_STATUS_SUCCESS; -} - -rpc_status_t rpc_server_init(uint32_t device_id, uint64_t num_ports, +rpc_status_t rpc_server_init(rpc_device_t *rpc_device, uint64_t num_ports, uint32_t lane_size, rpc_alloc_ty alloc, void *data) { - if (!state) - return RPC_STATUS_NOT_INITIALIZED; - if (device_id >= state->num_devices) - return RPC_STATUS_OUT_OF_RANGE; + if (!rpc_device) + return RPC_STATUS_ERROR; if (lane_size != 1 && lane_size != 32 && lane_size != 64) return RPC_STATUS_INVALID_LANE_SIZE; - if (!state->devices[device_id]) { - uint64_t size = rpc::Server::allocation_size(lane_size, num_ports); - void *buffer = alloc(size, data); + uint64_t size = rpc::Server::allocation_size(lane_size, num_ports); + void *buffer = alloc(size, data); - if (!buffer) - return RPC_STATUS_ERROR; + if (!buffer) + return RPC_STATUS_ERROR; - state->devices[device_id] = - std::make_unique(lane_size, num_ports, buffer); - if (!state->devices[device_id]) - return RPC_STATUS_ERROR; - } + Device *device = new Device(lane_size, num_ports, buffer); + if (!device) + return RPC_STATUS_ERROR; + rpc_device->handle = reinterpret_cast(device); return RPC_STATUS_SUCCESS; } -rpc_status_t rpc_server_shutdown(uint32_t device_id, rpc_free_ty dealloc, +rpc_status_t rpc_server_shutdown(rpc_device_t rpc_device, rpc_free_ty dealloc, void *data) { - if (!state) - return RPC_STATUS_NOT_INITIALIZED; - if (device_id >= state->num_devices) - return RPC_STATUS_OUT_OF_RANGE; - if (!state->devices[device_id]) + if (!rpc_device.handle) return RPC_STATUS_ERROR; - dealloc(state->devices[device_id]->buffer, data); - if (state->devices[device_id]) - state->devices[device_id].release(); + Device *device = reinterpret_cast(rpc_device.handle); + dealloc(device->buffer, data); + delete device; return RPC_STATUS_SUCCESS; } -rpc_status_t rpc_handle_server(uint32_t device_id) { - if (!state) - return RPC_STATUS_NOT_INITIALIZED; - if (device_id >= state->num_devices) - return RPC_STATUS_OUT_OF_RANGE; - if (!state->devices[device_id]) +rpc_status_t rpc_handle_server(rpc_device_t rpc_device) { + if (!rpc_device.handle) return RPC_STATUS_ERROR; + Device *device = reinterpret_cast(rpc_device.handle); uint32_t index = 0; for (;;) { - Device &device = *state->devices[device_id]; - rpc_status_t status = device.handle_server(index); + rpc_status_t status = device->handle_server(index); if (status != RPC_STATUS_CONTINUE) return status; } } -rpc_status_t rpc_register_callback(uint32_t device_id, uint16_t opcode, +rpc_status_t rpc_register_callback(rpc_device_t rpc_device, uint16_t opcode, rpc_opcode_callback_ty callback, void *data) { - if (!state) - return RPC_STATUS_NOT_INITIALIZED; - if (device_id >= state->num_devices) - return RPC_STATUS_OUT_OF_RANGE; - if (!state->devices[device_id]) + if (!rpc_device.handle) return RPC_STATUS_ERROR; - state->devices[device_id]->callbacks[opcode] = callback; - state->devices[device_id]->callback_data[opcode] = data; + Device *device = reinterpret_cast(rpc_device.handle); + + device->callbacks[opcode] = callback; + device->callback_data[opcode] = data; return RPC_STATUS_SUCCESS; } -const void *rpc_get_client_buffer(uint32_t device_id) { - if (!state || device_id >= state->num_devices || !state->devices[device_id]) +const void *rpc_get_client_buffer(rpc_device_t rpc_device) { + if (!rpc_device.handle) return nullptr; - return &state->devices[device_id]->client; + Device *device = reinterpret_cast(rpc_device.handle); + return &device->client; } uint64_t rpc_get_client_size() { return sizeof(rpc::Client); } -using ServerPort = std::variant; - -ServerPort get_port(rpc_port_t ref) { - return reinterpret_cast(ref.handle); -} - void rpc_send(rpc_port_t ref, rpc_port_callback_ty callback, void *data) { auto port = reinterpret_cast(ref.handle); port->send([=](rpc::Buffer *buffer) { diff --git a/openmp/libomptarget/plugins-nextgen/common/include/RPC.h b/openmp/libomptarget/plugins-nextgen/common/include/RPC.h index 2e39b3f299c8..b621cc0da458 100644 --- a/openmp/libomptarget/plugins-nextgen/common/include/RPC.h +++ b/openmp/libomptarget/plugins-nextgen/common/include/RPC.h @@ -16,6 +16,7 @@ #ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_RPC_H #define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_RPC_H +#include "llvm/ADT/DenseMap.h" #include "llvm/Support/Error.h" #include @@ -32,8 +33,6 @@ class DeviceImageTy; /// these routines will perform no action. struct RPCServerTy { public: - RPCServerTy(uint32_t NumDevices); - /// Check if this device image is using an RPC server. This checks for the /// precense of an externally visible symbol in the device image that will /// be present whenever RPC code is called. @@ -56,7 +55,9 @@ public: /// memory associated with the k llvm::Error deinitDevice(plugin::GenericDeviceTy &Device); - ~RPCServerTy(); +private: + /// Array from this device's identifier to its attached devices. + llvm::SmallVector Handles; }; } // namespace llvm::omp::target diff --git a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp index a4e6c9319215..55e2865d6aae 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp @@ -1492,7 +1492,7 @@ Error GenericPluginTy::init() { GlobalHandler = createGlobalHandler(); assert(GlobalHandler && "Invalid global handler"); - RPCServer = new RPCServerTy(NumDevices); + RPCServer = new RPCServerTy(); assert(RPCServer && "Invalid RPC server"); return Plugin::success(); diff --git a/openmp/libomptarget/plugins-nextgen/common/src/RPC.cpp b/openmp/libomptarget/plugins-nextgen/common/src/RPC.cpp index f46b27701b5b..fab0f6838f4a 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/RPC.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/RPC.cpp @@ -21,14 +21,6 @@ using namespace llvm; using namespace omp; using namespace target; -RPCServerTy::RPCServerTy(uint32_t NumDevices) { -#ifdef LIBOMPTARGET_RPC_SUPPORT - // If this fails then something is catastrophically wrong, just exit. - if (rpc_status_t Err = rpc_init(NumDevices)) - FATAL_MESSAGE(1, "Error initializing the RPC server: %d\n", Err); -#endif -} - llvm::Expected RPCServerTy::isDeviceUsingRPC(plugin::GenericDeviceTy &Device, plugin::GenericGlobalHandlerTy &Handler, @@ -44,7 +36,6 @@ Error RPCServerTy::initDevice(plugin::GenericDeviceTy &Device, plugin::GenericGlobalHandlerTy &Handler, plugin::DeviceImageTy &Image) { #ifdef LIBOMPTARGET_RPC_SUPPORT - uint32_t DeviceId = Device.getDeviceId(); auto Alloc = [](uint64_t Size, void *Data) { plugin::GenericDeviceTy &Device = *reinterpret_cast(Data); @@ -52,10 +43,12 @@ Error RPCServerTy::initDevice(plugin::GenericDeviceTy &Device, }; uint64_t NumPorts = std::min(Device.requestedRPCPortCount(), RPC_MAXIMUM_PORT_COUNT); - if (rpc_status_t Err = rpc_server_init(DeviceId, NumPorts, + rpc_device_t RPCDevice; + if (rpc_status_t Err = rpc_server_init(&RPCDevice, NumPorts, Device.getWarpSize(), Alloc, &Device)) return plugin::Plugin::error( - "Failed to initialize RPC server for device %d: %d", DeviceId, Err); + "Failed to initialize RPC server for device %d: %d", + Device.getDeviceId(), Err); // Register a custom opcode handler to perform plugin specific allocation. auto MallocHandler = [](rpc_port_t Port, void *Data) { @@ -70,10 +63,10 @@ Error RPCServerTy::initDevice(plugin::GenericDeviceTy &Device, Data); }; if (rpc_status_t Err = - rpc_register_callback(DeviceId, RPC_MALLOC, MallocHandler, &Device)) + rpc_register_callback(RPCDevice, RPC_MALLOC, MallocHandler, &Device)) return plugin::Plugin::error( - "Failed to register RPC malloc handler for device %d: %d\n", DeviceId, - Err); + "Failed to register RPC malloc handler for device %d: %d\n", + Device.getDeviceId(), Err); // Register a custom opcode handler to perform plugin specific deallocation. auto FreeHandler = [](rpc_port_t Port, void *Data) { @@ -88,10 +81,10 @@ Error RPCServerTy::initDevice(plugin::GenericDeviceTy &Device, Data); }; if (rpc_status_t Err = - rpc_register_callback(DeviceId, RPC_FREE, FreeHandler, &Device)) + rpc_register_callback(RPCDevice, RPC_FREE, FreeHandler, &Device)) return plugin::Plugin::error( - "Failed to register RPC free handler for device %d: %d\n", DeviceId, - Err); + "Failed to register RPC free handler for device %d: %d\n", + Device.getDeviceId(), Err); // Get the address of the RPC client from the device. void *ClientPtr; @@ -104,17 +97,20 @@ Error RPCServerTy::initDevice(plugin::GenericDeviceTy &Device, sizeof(void *), nullptr)) return Err; - const void *ClientBuffer = rpc_get_client_buffer(DeviceId); + const void *ClientBuffer = rpc_get_client_buffer(RPCDevice); if (auto Err = Device.dataSubmit(ClientPtr, ClientBuffer, rpc_get_client_size(), nullptr)) return Err; + Handles.resize(Device.getDeviceId() + 1); + Handles[Device.getDeviceId()] = RPCDevice.handle; #endif return Error::success(); } Error RPCServerTy::runServer(plugin::GenericDeviceTy &Device) { #ifdef LIBOMPTARGET_RPC_SUPPORT - if (rpc_status_t Err = rpc_handle_server(Device.getDeviceId())) + rpc_device_t RPCDevice{Handles[Device.getDeviceId()]}; + if (rpc_status_t Err = rpc_handle_server(RPCDevice)) return plugin::Plugin::error( "Error while running RPC server on device %d: %d", Device.getDeviceId(), Err); @@ -124,22 +120,16 @@ Error RPCServerTy::runServer(plugin::GenericDeviceTy &Device) { Error RPCServerTy::deinitDevice(plugin::GenericDeviceTy &Device) { #ifdef LIBOMPTARGET_RPC_SUPPORT + rpc_device_t RPCDevice{Handles[Device.getDeviceId()]}; auto Dealloc = [](void *Ptr, void *Data) { plugin::GenericDeviceTy &Device = *reinterpret_cast(Data); Device.free(Ptr, TARGET_ALLOC_HOST); }; - if (rpc_status_t Err = - rpc_server_shutdown(Device.getDeviceId(), Dealloc, &Device)) + if (rpc_status_t Err = rpc_server_shutdown(RPCDevice, Dealloc, &Device)) return plugin::Plugin::error( "Failed to shut down RPC server for device %d: %d", Device.getDeviceId(), Err); #endif return Error::success(); } - -RPCServerTy::~RPCServerTy() { -#ifdef LIBOMPTARGET_RPC_SUPPORT - rpc_shutdown(); -#endif -} -- GitLab From 99379522d00d0720bcda5067d4de2bfb0c279f15 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 29 Mar 2024 13:13:03 -0500 Subject: [PATCH 0016/1690] [libc][Docs] Update RPC server example using CUDA after changes Summary: This has changed, so update it to match the new interface. --- libc/docs/gpu/rpc.rst | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/libc/docs/gpu/rpc.rst b/libc/docs/gpu/rpc.rst index 9d6d8099db95..e13a377f305c 100644 --- a/libc/docs/gpu/rpc.rst +++ b/libc/docs/gpu/rpc.rst @@ -251,14 +251,10 @@ but the following example shows how it can be used by a standard user. __global__ void hello() { puts("Hello world!"); } int main() { - int device = 0; - // Initialize the RPC server to run on a single device. - if (rpc_status_t err = rpc_init(/*num_device=*/1)) - handle_error(err); - // Initialize the RPC server to run on the given device. + rpc_device_t device; if (rpc_status_t err = - rpc_server_init(device, RPC_MAXIMUM_PORT_COUNT, + rpc_server_init(&device, RPC_MAXIMUM_PORT_COUNT, /*warp_size=*/32, alloc_host, /*data=*/nullptr)) handle_error(err); @@ -277,6 +273,7 @@ but the following example shows how it can be used by a standard user. hello<<<1, 1, 0, stream>>>(); // While the kernel is executing, check the RPC server for work to do. + // Requires non-blocking CUDA kernels but avoids a separate thread. while (cudaStreamQuery(stream) == cudaErrorNotReady) if (rpc_status_t err = rpc_handle_server(device)) handle_error(err); @@ -286,10 +283,6 @@ but the following example shows how it can be used by a standard user. rpc_server_shutdown(device, free_host, /*data=*/nullptr)) handle_error(err); - // Shut down the entire RPC server interface. - if (rpc_status_t err = rpc_shutdown()) - handle_error(err); - return EXIT_SUCCESS; } @@ -300,7 +293,7 @@ associated with relocatable device code linking. .. code-block:: sh - $> clang++ -x cuda rpc.cpp --offload-arch=native -fgpu-rdc -lcudart -lcgpu \ + $> clang++ -x cuda rpc.cpp --offload-arch=native -fgpu-rdc -lcudart -lcgpu-nvptx \ -Iinclude -L/lib -lllvmlibc_rpc_server \ -O3 -foffload-lto -o hello $> ./hello -- GitLab From 407a2f231a81862e20d80059870c48d818b61ec2 Mon Sep 17 00:00:00 2001 From: Jan Svoboda Date: Fri, 29 Mar 2024 11:20:55 -0700 Subject: [PATCH 0017/1690] [clang] Move state out of `PreprocessorOptions` (1/n) (#86358) An instance of `PreprocessorOptions` is part of `CompilerInvocation` which is supposed to be a value type. The `DependencyDirectivesForFile` member is problematic, since it holds an owning reference of the scanning VFS. This makes it not a true value type, and it can keep potentially large chunk of memory (the local cache in the scanning VFS) alive for longer than clients might expect. Let's move it into the `Preprocessor` instead. --- .../include/clang/Frontend/FrontendActions.h | 14 ++++++-- clang/include/clang/Lex/Preprocessor.h | 18 ++++++++++ clang/include/clang/Lex/PreprocessorOptions.h | 13 ------- clang/lib/Frontend/FrontendActions.cpp | 7 +++- clang/lib/Lex/PPLexerChange.cpp | 12 ++----- .../DependencyScanningWorker.cpp | 36 ++++++++++--------- .../Lex/PPDependencyDirectivesTest.cpp | 11 +++--- 7 files changed, 65 insertions(+), 46 deletions(-) diff --git a/clang/include/clang/Frontend/FrontendActions.h b/clang/include/clang/Frontend/FrontendActions.h index a620ddfc4044..0518a8823a03 100644 --- a/clang/include/clang/Frontend/FrontendActions.h +++ b/clang/include/clang/Frontend/FrontendActions.h @@ -34,12 +34,18 @@ public: /// Preprocessor-based frontend action that also loads PCH files. class ReadPCHAndPreprocessAction : public FrontendAction { + llvm::unique_function AdjustCI; + void ExecuteAction() override; std::unique_ptr CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override; public: + ReadPCHAndPreprocessAction( + llvm::unique_function AdjustCI) + : AdjustCI(std::move(AdjustCI)) {} + bool usesPreprocessorOnly() const override { return false; } }; @@ -321,11 +327,15 @@ protected: class GetDependenciesByModuleNameAction : public PreprocessOnlyAction { StringRef ModuleName; + llvm::unique_function AdjustCI; + void ExecuteAction() override; public: - GetDependenciesByModuleNameAction(StringRef ModuleName) - : ModuleName(ModuleName) {} + GetDependenciesByModuleNameAction( + StringRef ModuleName, + llvm::unique_function AdjustCI) + : ModuleName(ModuleName), AdjustCI(std::move(AdjustCI)) {} }; } // end namespace clang diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 0836b7d439bb..24e146a589a7 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -736,6 +736,19 @@ private: State ConditionalStackState = Off; } PreambleConditionalStack; + /// Function for getting the dependency preprocessor directives of a file. + /// + /// These are directives derived from a special form of lexing where the + /// source input is scanned for the preprocessor directives that might have an + /// effect on the dependencies for a compilation unit. + /// + /// Enables a client to cache the directives for a file and provide them + /// across multiple compiler invocations. + /// FIXME: Allow returning an error. + using DependencyDirectivesFn = llvm::unique_function>(FileEntryRef)>; + DependencyDirectivesFn DependencyDirectivesForFile; + /// The current top of the stack that we're lexing from if /// not expanding a macro and we are lexing directly from source code. /// @@ -1270,6 +1283,11 @@ public: /// false if it is producing tokens to be consumed by Parse and Sema. bool isPreprocessedOutput() const { return PreprocessedOutput; } + /// Set the function used to get dependency directives for a file. + void setDependencyDirectivesFn(DependencyDirectivesFn Fn) { + DependencyDirectivesForFile = std::move(Fn); + } + /// Return true if we are lexing directly from the specified lexer. bool isCurrentLexer(const PreprocessorLexer *L) const { return CurPPLexer == L; diff --git a/clang/include/clang/Lex/PreprocessorOptions.h b/clang/include/clang/Lex/PreprocessorOptions.h index f841e4a028df..24e0ca61d414 100644 --- a/clang/include/clang/Lex/PreprocessorOptions.h +++ b/clang/include/clang/Lex/PreprocessorOptions.h @@ -208,19 +208,6 @@ public: /// build it again. std::shared_ptr FailedModules; - /// Function for getting the dependency preprocessor directives of a file. - /// - /// These are directives derived from a special form of lexing where the - /// source input is scanned for the preprocessor directives that might have an - /// effect on the dependencies for a compilation unit. - /// - /// Enables a client to cache the directives for a file and provide them - /// across multiple compiler invocations. - /// FIXME: Allow returning an error. - std::function>( - FileEntryRef)> - DependencyDirectivesForFile; - /// Set up preprocessor for RunAnalysis action. bool SetUpStaticAnalyzer = false; diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 3fd1cdd3b479..0bc26b694cfc 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -69,7 +69,10 @@ void InitOnlyAction::ExecuteAction() { // Basically PreprocessOnlyAction::ExecuteAction. void ReadPCHAndPreprocessAction::ExecuteAction() { - Preprocessor &PP = getCompilerInstance().getPreprocessor(); + CompilerInstance &CI = getCompilerInstance(); + AdjustCI(CI); + + Preprocessor &PP = CI.getPreprocessor(); // Ignore unknown pragmas. PP.IgnorePragmas(); @@ -1188,6 +1191,8 @@ void PrintDependencyDirectivesSourceMinimizerAction::ExecuteAction() { void GetDependenciesByModuleNameAction::ExecuteAction() { CompilerInstance &CI = getCompilerInstance(); + AdjustCI(CI); + Preprocessor &PP = CI.getPreprocessor(); SourceManager &SM = PP.getSourceManager(); FileID MainFileID = SM.getMainFileID(); diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp index 3b1b6df1dbae..a0cc2b516574 100644 --- a/clang/lib/Lex/PPLexerChange.cpp +++ b/clang/lib/Lex/PPLexerChange.cpp @@ -93,16 +93,10 @@ bool Preprocessor::EnterSourceFile(FileID FID, ConstSearchDirIterator CurDir, } Lexer *TheLexer = new Lexer(FID, *InputFile, *this, IsFirstIncludeOfFile); - if (getPreprocessorOpts().DependencyDirectivesForFile && - FID != PredefinesFileID) { - if (OptionalFileEntryRef File = SourceMgr.getFileEntryRefForID(FID)) { - if (std::optional> - DepDirectives = - getPreprocessorOpts().DependencyDirectivesForFile(*File)) { + if (DependencyDirectivesForFile && FID != PredefinesFileID) + if (OptionalFileEntryRef File = SourceMgr.getFileEntryRefForID(FID)) + if (auto DepDirectives = DependencyDirectivesForFile(*File)) TheLexer->DepDirectives = *DepDirectives; - } - } - } EnterSourceFileWithLexer(TheLexer, CurDir); return false; diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 33b43417a661..492b8f1e2b38 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -363,20 +363,22 @@ public: PrebuiltModuleVFSMap, ScanInstance.getDiagnostics())) return false; - // Use the dependency scanning optimized file system if requested to do so. - if (DepFS) { - llvm::IntrusiveRefCntPtr LocalDepFS = - DepFS; - ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile = - [LocalDepFS = std::move(LocalDepFS)](FileEntryRef File) - -> std::optional> { - if (llvm::ErrorOr Entry = - LocalDepFS->getOrCreateFileSystemEntry(File.getName())) - if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry)) - return Entry->getDirectiveTokens(); - return std::nullopt; - }; - } + auto AdjustCI = [&](CompilerInstance &CI) { + // Set up the dependency scanning file system callback if requested. + if (DepFS) { + auto GetDependencyDirectives = [LocalDepFS = DepFS](FileEntryRef File) + -> std::optional> { + if (llvm::ErrorOr Entry = + LocalDepFS->getOrCreateFileSystemEntry(File.getName())) + if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry)) + return Entry->getDirectiveTokens(); + return std::nullopt; + }; + + CI.getPreprocessor().setDependencyDirectivesFn( + std::move(GetDependencyDirectives)); + } + }; // Create the dependency collector that will collect the produced // dependencies. @@ -428,9 +430,11 @@ public: std::unique_ptr Action; if (ModuleName) - Action = std::make_unique(*ModuleName); + Action = std::make_unique( + *ModuleName, std::move(AdjustCI)); else - Action = std::make_unique(); + Action = + std::make_unique(std::move(AdjustCI)); if (ScanInstance.getDiagnostics().hasErrorOccurred()) return false; diff --git a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp index 6ff87f720a55..0c396720ece6 100644 --- a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp +++ b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp @@ -117,11 +117,6 @@ TEST_F(PPDependencyDirectivesTest, MacroGuard) { }; auto PPOpts = std::make_shared(); - PPOpts->DependencyDirectivesForFile = [&](FileEntryRef File) - -> std::optional> { - return getDependencyDirectives(File); - }; - TrivialModuleLoader ModLoader; HeaderSearch HeaderInfo(std::make_shared(), SourceMgr, Diags, LangOpts, Target.get()); @@ -130,6 +125,12 @@ TEST_F(PPDependencyDirectivesTest, MacroGuard) { /*OwnsHeaderSearch =*/false); PP.Initialize(*Target); + PP.setDependencyDirectivesFn( + [&](FileEntryRef File) + -> std::optional> { + return getDependencyDirectives(File); + }); + SmallVector IncludedFiles; PP.addPPCallbacks(std::make_unique(PP, IncludedFiles)); PP.EnterMainSourceFile(); -- GitLab From beaff78528747dfbd50f9f4df77ac25f459075be Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Fri, 29 Mar 2024 19:29:54 +0100 Subject: [PATCH 0018/1690] [libc++] Optimize the std::mismatch tail (#83440) This adds vectorization to the last 0-3 vectors and, if the range is large enough, the remaining elements that don't fill a vector completely. ``` ----------------------------------------------------------------------- Benchmark old full vectors partial vector ----------------------------------------------------------------------- bm_mismatch/1 1.40 ns 1.62 ns 2.09 ns bm_mismatch/2 1.88 ns 2.10 ns 2.33 ns bm_mismatch/3 2.67 ns 2.56 ns 2.72 ns bm_mismatch/4 3.01 ns 3.20 ns 3.70 ns bm_mismatch/5 3.51 ns 3.73 ns 3.64 ns bm_mismatch/6 4.71 ns 4.85 ns 4.37 ns bm_mismatch/7 5.12 ns 5.33 ns 4.37 ns bm_mismatch/8 5.79 ns 6.02 ns 4.75 ns bm_mismatch/15 9.20 ns 10.5 ns 7.23 ns bm_mismatch/16 10.2 ns 10.1 ns 7.46 ns bm_mismatch/17 10.2 ns 10.8 ns 7.57 ns bm_mismatch/31 17.6 ns 17.1 ns 10.8 ns bm_mismatch/32 17.4 ns 1.64 ns 1.64 ns bm_mismatch/33 23.3 ns 2.10 ns 2.33 ns bm_mismatch/63 31.8 ns 16.9 ns 2.33 ns bm_mismatch/64 32.6 ns 2.10 ns 2.10 ns bm_mismatch/65 33.6 ns 2.57 ns 2.80 ns bm_mismatch/127 67.3 ns 18.1 ns 3.27 ns bm_mismatch/128 2.17 ns 2.14 ns 2.57 ns bm_mismatch/129 2.36 ns 2.80 ns 3.27 ns bm_mismatch/255 67.5 ns 19.6 ns 4.68 ns bm_mismatch/256 3.76 ns 3.71 ns 3.97 ns bm_mismatch/257 3.77 ns 4.04 ns 4.43 ns bm_mismatch/511 70.8 ns 22.1 ns 7.47 ns bm_mismatch/512 7.27 ns 7.30 ns 6.95 ns bm_mismatch/513 7.11 ns 7.05 ns 6.96 ns bm_mismatch/1023 75.9 ns 27.4 ns 13.3 ns bm_mismatch/1024 13.9 ns 13.8 ns 12.4 ns bm_mismatch/1025 13.6 ns 13.6 ns 12.8 ns bm_mismatch/2047 87.3 ns 37.5 ns 25.4 ns bm_mismatch/2048 26.8 ns 27.4 ns 24.0 ns bm_mismatch/2049 26.7 ns 27.3 ns 25.5 ns bm_mismatch/4095 112 ns 64.7 ns 48.7 ns bm_mismatch/4096 53.0 ns 54.2 ns 46.8 ns bm_mismatch/4097 52.7 ns 54.2 ns 48.4 ns bm_mismatch/8191 160 ns 118 ns 98.4 ns bm_mismatch/8192 107 ns 108 ns 96.0 ns bm_mismatch/8193 106 ns 108 ns 97.2 ns bm_mismatch/16383 283 ns 234 ns 215 ns bm_mismatch/16384 227 ns 223 ns 217 ns bm_mismatch/16385 221 ns 221 ns 215 ns bm_mismatch/32767 547 ns 499 ns 488 ns bm_mismatch/32768 495 ns 492 ns 492 ns bm_mismatch/32769 491 ns 489 ns 488 ns bm_mismatch/65535 1028 ns 979 ns 971 ns bm_mismatch/65536 976 ns 970 ns 974 ns bm_mismatch/65537 970 ns 965 ns 971 ns bm_mismatch/131071 2031 ns 1948 ns 2005 ns bm_mismatch/131072 1973 ns 1955 ns 1974 ns bm_mismatch/131073 1989 ns 1932 ns 2001 ns bm_mismatch/262143 4469 ns 4244 ns 4223 ns bm_mismatch/262144 4443 ns 4183 ns 4243 ns bm_mismatch/262145 4400 ns 4232 ns 4246 ns bm_mismatch/524287 10169 ns 9733 ns 9592 ns bm_mismatch/524288 10154 ns 9664 ns 9843 ns bm_mismatch/524289 10113 ns 9641 ns 10003 ns bm_mismatch/1 1.86 ns 2.53 ns 2.32 ns bm_mismatch/2 2.57 ns 2.77 ns 2.55 ns bm_mismatch/3 3.26 ns 3.00 ns 2.79 ns bm_mismatch/4 3.95 ns 3.39 ns 3.15 ns bm_mismatch/5 4.83 ns 3.97 ns 3.72 ns bm_mismatch/6 5.43 ns 4.34 ns 4.03 ns bm_mismatch/7 6.11 ns 4.73 ns 4.44 ns bm_mismatch/8 6.84 ns 5.02 ns 4.79 ns bm_mismatch/15 11.5 ns 7.12 ns 6.50 ns bm_mismatch/16 13.9 ns 1.87 ns 2.11 ns bm_mismatch/17 14.0 ns 3.00 ns 2.47 ns bm_mismatch/31 23.1 ns 7.87 ns 2.47 ns bm_mismatch/32 23.8 ns 2.57 ns 2.81 ns bm_mismatch/33 24.5 ns 3.70 ns 2.94 ns bm_mismatch/63 44.8 ns 9.37 ns 3.46 ns bm_mismatch/64 2.32 ns 2.57 ns 2.64 ns bm_mismatch/65 2.52 ns 3.02 ns 3.51 ns bm_mismatch/127 45.6 ns 9.97 ns 5.18 ns bm_mismatch/128 3.85 ns 3.93 ns 3.94 ns bm_mismatch/129 3.82 ns 4.20 ns 4.70 ns bm_mismatch/255 50.4 ns 12.6 ns 8.07 ns bm_mismatch/256 7.23 ns 6.91 ns 6.98 ns bm_mismatch/257 7.24 ns 7.19 ns 7.55 ns bm_mismatch/511 52.3 ns 17.8 ns 14.0 ns bm_mismatch/512 13.6 ns 13.7 ns 13.6 ns bm_mismatch/513 13.9 ns 13.8 ns 18.5 ns bm_mismatch/1023 60.9 ns 30.9 ns 26.3 ns bm_mismatch/1024 26.7 ns 27.7 ns 25.7 ns bm_mismatch/1025 27.7 ns 27.6 ns 25.3 ns bm_mismatch/2047 88.4 ns 58.0 ns 51.6 ns bm_mismatch/2048 52.8 ns 55.3 ns 50.6 ns bm_mismatch/2049 55.2 ns 54.8 ns 48.7 ns bm_mismatch/4095 153 ns 113 ns 102 ns bm_mismatch/4096 105 ns 110 ns 101 ns bm_mismatch/4097 110 ns 110 ns 99.1 ns bm_mismatch/8191 277 ns 219 ns 206 ns bm_mismatch/8192 226 ns 214 ns 250 ns bm_mismatch/8193 226 ns 207 ns 208 ns bm_mismatch/16383 519 ns 492 ns 488 ns bm_mismatch/16384 494 ns 492 ns 492 ns bm_mismatch/16385 492 ns 488 ns 489 ns bm_mismatch/32767 1007 ns 968 ns 964 ns bm_mismatch/32768 977 ns 972 ns 970 ns bm_mismatch/32769 972 ns 962 ns 967 ns bm_mismatch/65535 1978 ns 1918 ns 1956 ns bm_mismatch/65536 1940 ns 1927 ns 1970 ns bm_mismatch/65537 1937 ns 1922 ns 1959 ns bm_mismatch/131071 4524 ns 4193 ns 4304 ns bm_mismatch/131072 4445 ns 4196 ns 4306 ns bm_mismatch/131073 4452 ns 4278 ns 4311 ns bm_mismatch/262143 9801 ns 10188 ns 9634 ns bm_mismatch/262144 9738 ns 10151 ns 9651 ns bm_mismatch/262145 9716 ns 10171 ns 9715 ns bm_mismatch/524287 19944 ns 20718 ns 20044 ns bm_mismatch/524288 21139 ns 20647 ns 20008 ns bm_mismatch/524289 21162 ns 19512 ns 20068 ns bm_mismatch/1 1.40 ns 1.84 ns 1.87 ns bm_mismatch/2 1.87 ns 2.08 ns 2.09 ns bm_mismatch/3 2.36 ns 2.31 ns 2.87 ns bm_mismatch/4 3.06 ns 2.72 ns 2.95 ns bm_mismatch/5 3.66 ns 3.37 ns 3.42 ns bm_mismatch/6 4.55 ns 3.65 ns 3.73 ns bm_mismatch/7 5.03 ns 3.93 ns 3.94 ns bm_mismatch/8 5.67 ns 1.86 ns 1.87 ns bm_mismatch/15 9.89 ns 4.41 ns 2.34 ns bm_mismatch/16 10.1 ns 2.33 ns 2.34 ns bm_mismatch/17 10.2 ns 3.34 ns 2.86 ns bm_mismatch/31 17.2 ns 5.54 ns 3.28 ns bm_mismatch/32 2.16 ns 2.15 ns 2.58 ns bm_mismatch/33 2.36 ns 3.01 ns 3.28 ns bm_mismatch/63 17.7 ns 6.50 ns 4.93 ns bm_mismatch/64 3.81 ns 3.58 ns 3.90 ns bm_mismatch/65 3.74 ns 4.36 ns 4.45 ns bm_mismatch/127 19.5 ns 9.56 ns 7.74 ns bm_mismatch/128 7.30 ns 6.41 ns 6.85 ns bm_mismatch/129 7.09 ns 7.04 ns 7.06 ns bm_mismatch/255 24.7 ns 14.8 ns 13.3 ns bm_mismatch/256 14.0 ns 12.1 ns 12.3 ns bm_mismatch/257 13.8 ns 12.7 ns 12.8 ns bm_mismatch/511 34.3 ns 26.3 ns 24.8 ns bm_mismatch/512 27.6 ns 23.6 ns 23.9 ns bm_mismatch/513 27.3 ns 24.4 ns 25.1 ns bm_mismatch/1023 62.5 ns 50.9 ns 48.3 ns bm_mismatch/1024 54.4 ns 46.1 ns 46.6 ns bm_mismatch/1025 54.2 ns 48.4 ns 47.5 ns bm_mismatch/2047 116 ns 97.8 ns 94.1 ns bm_mismatch/2048 108 ns 92.6 ns 92.4 ns bm_mismatch/2049 108 ns 104 ns 94.0 ns bm_mismatch/4095 233 ns 222 ns 205 ns bm_mismatch/4096 226 ns 223 ns 225 ns bm_mismatch/4097 221 ns 219 ns 210 ns bm_mismatch/8191 499 ns 485 ns 488 ns bm_mismatch/8192 496 ns 490 ns 495 ns bm_mismatch/8193 491 ns 485 ns 488 ns bm_mismatch/16383 982 ns 962 ns 964 ns bm_mismatch/16384 974 ns 971 ns 971 ns bm_mismatch/16385 971 ns 961 ns 968 ns bm_mismatch/32767 2003 ns 1959 ns 1920 ns bm_mismatch/32768 1996 ns 1947 ns 1928 ns bm_mismatch/32769 1990 ns 1945 ns 1926 ns bm_mismatch/65535 4434 ns 4275 ns 4312 ns bm_mismatch/65536 4437 ns 4267 ns 4321 ns bm_mismatch/65537 4442 ns 4261 ns 4321 ns bm_mismatch/131071 9673 ns 9648 ns 9465 ns bm_mismatch/131072 9667 ns 9671 ns 9465 ns bm_mismatch/131073 9661 ns 9653 ns 9464 ns bm_mismatch/262143 20595 ns 19605 ns 19064 ns bm_mismatch/262144 19894 ns 19572 ns 19009 ns bm_mismatch/262145 19851 ns 19656 ns 18999 ns bm_mismatch/524287 39556 ns 39364 ns 38131 ns bm_mismatch/524288 39678 ns 39573 ns 38183 ns bm_mismatch/524289 40168 ns 39301 ns 38121 ns ``` --- .../benchmarks/algorithms/mismatch.bench.cpp | 15 ++++++++-- libcxx/include/__algorithm/mismatch.h | 29 ++++++++++++++++++- .../mismatch/mismatch.pass.cpp | 28 ++++++++++++++++++ 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/libcxx/benchmarks/algorithms/mismatch.bench.cpp b/libcxx/benchmarks/algorithms/mismatch.bench.cpp index 9274932a764c..06289068bb04 100644 --- a/libcxx/benchmarks/algorithms/mismatch.bench.cpp +++ b/libcxx/benchmarks/algorithms/mismatch.bench.cpp @@ -10,6 +10,15 @@ #include #include +void BenchmarkSizes(benchmark::internal::Benchmark* Benchmark) { + Benchmark->DenseRange(1, 8); + for (size_t i = 16; i != 1 << 20; i *= 2) { + Benchmark->Arg(i - 1); + Benchmark->Arg(i); + Benchmark->Arg(i + 1); + } +} + // TODO: Look into benchmarking aligned and unaligned memory explicitly // (currently things happen to be aligned because they are malloced that way) template @@ -24,8 +33,8 @@ static void bm_mismatch(benchmark::State& state) { benchmark::DoNotOptimize(std::mismatch(vec1.begin(), vec1.end(), vec2.begin())); } } -BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); -BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); -BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); +BENCHMARK(bm_mismatch)->Apply(BenchmarkSizes); +BENCHMARK(bm_mismatch)->Apply(BenchmarkSizes); +BENCHMARK(bm_mismatch)->Apply(BenchmarkSizes); BENCHMARK_MAIN(); diff --git a/libcxx/include/__algorithm/mismatch.h b/libcxx/include/__algorithm/mismatch.h index 4eb693a1f2e9..d933a84cada9 100644 --- a/libcxx/include/__algorithm/mismatch.h +++ b/libcxx/include/__algorithm/mismatch.h @@ -64,7 +64,10 @@ __mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __ constexpr size_t __unroll_count = 4; constexpr size_t __vec_size = __native_vector_size<_Tp>; using __vec = __simd_vector<_Tp, __vec_size>; + if (!__libcpp_is_constant_evaluated()) { + auto __orig_first1 = __first1; + auto __last2 = __first2 + (__last1 - __first1); while (static_cast(__last1 - __first1) >= __unroll_count * __vec_size) [[__unlikely__]] { __vec __lhs[__unroll_count]; __vec __rhs[__unroll_count]; @@ -84,8 +87,32 @@ __mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __ __first1 += __unroll_count * __vec_size; __first2 += __unroll_count * __vec_size; } + + // check the remaining 0-3 vectors + while (static_cast(__last1 - __first1) >= __vec_size) { + if (auto __cmp_res = std::__load_vector<__vec>(__first1) == std::__load_vector<__vec>(__first2); + !std::__all_of(__cmp_res)) { + auto __offset = std::__find_first_not_set(__cmp_res); + return {__first1 + __offset, __first2 + __offset}; + } + __first1 += __vec_size; + __first2 += __vec_size; + } + + if (__last1 - __first1 == 0) + return {__first1, __first2}; + + // Check if we can load elements in front of the current pointer. If that's the case load a vector at + // (last - vector_size) to check the remaining elements + if (static_cast(__first1 - __orig_first1) >= __vec_size) { + __first1 = __last1 - __vec_size; + __first2 = __last2 - __vec_size; + auto __offset = + std::__find_first_not_set(std::__load_vector<__vec>(__first1) == std::__load_vector<__vec>(__first2)); + return {__first1 + __offset, __first2 + __offset}; + } // else loop over the elements individually } - // TODO: Consider vectorizing the tail + return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2); } diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp index e7f3994d977d..55c9eea863c3 100644 --- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp +++ b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp @@ -184,5 +184,33 @@ int main(int, char**) { } } + { // check the tail of the vectorized loop + for (size_t vec_size = 1; vec_size != 256; ++vec_size) { + { + std::vector lhs(256); + std::vector rhs(256); + + check(lhs, rhs, lhs.size()); + lhs.back() = 1; + check(lhs, rhs, lhs.size() - 1); + lhs.back() = 0; + rhs.back() = 1; + check(lhs, rhs, lhs.size() - 1); + rhs.back() = 0; + } + { + std::vector lhs(256); + std::vector rhs(256); + + check(lhs, rhs, lhs.size()); + lhs.back() = 1; + check(lhs, rhs, lhs.size() - 1); + lhs.back() = 0; + rhs.back() = 1; + check(lhs, rhs, lhs.size() - 1); + rhs.back() = 0; + } + } + } return 0; } -- GitLab From 2684a0966d5c84071c811c68ec25e41a0beb26f4 Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Fri, 29 Mar 2024 19:33:46 +0100 Subject: [PATCH 0019/1690] [libc++] Document guidelines for applying [[nodiscard]] (#84000) We've been applying ``[[nodiscard]]`` more liberally recently, but we don't have any documented guidance on when it's correct to add it. This patch adds that guidance. Follow-up patches will gradually apply it to the code base. --- libcxx/docs/DesignDocs/NodiscardPolicy.rst | 42 ++++++++++++++++++++++ libcxx/docs/index.rst | 1 + 2 files changed, 43 insertions(+) create mode 100644 libcxx/docs/DesignDocs/NodiscardPolicy.rst diff --git a/libcxx/docs/DesignDocs/NodiscardPolicy.rst b/libcxx/docs/DesignDocs/NodiscardPolicy.rst new file mode 100644 index 000000000000..afbb18b0096d --- /dev/null +++ b/libcxx/docs/DesignDocs/NodiscardPolicy.rst @@ -0,0 +1,42 @@ +=================================================== +Guidelines for applying ``[[nodiscard]]`` in libc++ +=================================================== + +Libc++ adds ``[[nodiscard]]`` to functions in a lot of places. The standards +committee has decided to not have a recommended practice where to put them, so +this document lists where ``[[nodiscard]]`` should be applied in libc++. + +When should ``[[nodiscard]]`` be added to functions? +==================================================== + +``[[nodiscard]]`` should be applied to functions + +- where discarding the return value is most likely a correctness issue. + For example a locking constructor in ``unique_lock``. + +- where discarding the return value likely points to the user wanting to do + something different. For example ``vector::empty()``, which probably should + have been ``vector::clear()``. + + This can help spotting bugs easily which otherwise may take a very long time + to find. + +- which return a constant. For example ``numeric_limits::min()``. +- which only observe a value. For example ``string::size()``. + + Code that discards values from these kinds of functions is dead code. It can + either be removed, or the programmer meant to do something different. + +- where discarding the value is most likely a misuse of the function. For + example ``find``. + + This protects programmers from assuming too much about how the internals of + a function work, making code more robust in the presence of future + optimizations. + +What should be done when adding ``[[nodiscard]]`` to a function? +================================================================ + +Applications of ``[[nodiscard]]`` are code like any other code, so we aim to +test them. This can be done with a ``.verify.cpp`` test. Many examples are +available. Just look for tests with the suffix ``.nodiscard.verify.cpp``. diff --git a/libcxx/docs/index.rst b/libcxx/docs/index.rst index aa1bd4b83b26..2a7e47dfe6d8 100644 --- a/libcxx/docs/index.rst +++ b/libcxx/docs/index.rst @@ -189,6 +189,7 @@ Design Documents DesignDocs/FeatureTestMacros DesignDocs/FileTimeType DesignDocs/HeaderRemovalPolicy + DesignDocs/NodiscardPolicy DesignDocs/NoexceptPolicy DesignDocs/PSTLIntegration DesignDocs/ThreadingSupportAPI -- GitLab From fe893c93b7bd3b3c436b97383db21e08c9ec4b25 Mon Sep 17 00:00:00 2001 From: "Kevin P. Neal" Date: Fri, 29 Mar 2024 14:54:51 -0400 Subject: [PATCH 0020/1690] [FPEnv][AtomicExpand] Correct strictfp attribute handling in AtomicExpandPass (#87082) The AtomicExpand pass was lowering function calls with the strictfp attribute to sequences that included function calls incorrectly lacking the attribute. This patch corrects that. The pass now also emits the correct constrained fp call instead of normal FP instructions when in a function with the strictfp attribute. Test changes verified with D146845. --- llvm/lib/CodeGen/AtomicExpandPass.cpp | 3 +++ .../AMDGPU/expand-atomic-rmw-fadd.ll | 24 +++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp index 894285a7eb25..d5db79df6862 100644 --- a/llvm/lib/CodeGen/AtomicExpandPass.cpp +++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp @@ -135,10 +135,13 @@ public: // IRBuilder to be used for replacement atomic instructions. struct ReplacementIRBuilder : IRBuilder { // Preserves the DebugLoc from I, and preserves still valid metadata. + // Enable StrictFP builder mode when appropriate. explicit ReplacementIRBuilder(Instruction *I, const DataLayout &DL) : IRBuilder(I->getContext(), DL) { SetInsertPoint(I); this->CollectMetadataToCopy(I, {LLVMContext::MD_pcsections}); + if (BB->getParent()->getAttributes().hasFnAttr(Attribute::StrictFP)) + this->setIsFPConstrained(true); } }; diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll index b6e6b2602495..a5d4c329446f 100644 --- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll +++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll @@ -3028,7 +3028,7 @@ define bfloat @test_atomicrmw_fadd_bf16_global_system_align4(ptr addrspace(1) %p define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bfloat %value) #2 { ; CI-LABEL: @test_atomicrmw_fadd_bf16_local_strictfp( -; CI-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) +; CI-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) #[[ATTR6]] ; CI-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i32 ; CI-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3 ; CI-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3 @@ -3041,7 +3041,7 @@ define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf ; CI-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]] ; CI-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16 ; CI-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat -; CI-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]] +; CI-NEXT: [[NEW:%.*]] = call bfloat @llvm.experimental.constrained.fadd.bf16(bfloat [[TMP4]], bfloat [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR6]] ; CI-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16 ; CI-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32 ; CI-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]] @@ -3058,7 +3058,7 @@ define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf ; CI-NEXT: ret bfloat [[TMP7]] ; ; GFX9-LABEL: @test_atomicrmw_fadd_bf16_local_strictfp( -; GFX9-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) +; GFX9-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) #[[ATTR6]] ; GFX9-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i32 ; GFX9-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3 ; GFX9-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3 @@ -3071,7 +3071,7 @@ define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf ; GFX9-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]] ; GFX9-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16 ; GFX9-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat -; GFX9-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]] +; GFX9-NEXT: [[NEW:%.*]] = call bfloat @llvm.experimental.constrained.fadd.bf16(bfloat [[TMP4]], bfloat [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR6]] ; GFX9-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16 ; GFX9-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32 ; GFX9-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]] @@ -3088,7 +3088,7 @@ define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf ; GFX9-NEXT: ret bfloat [[TMP7]] ; ; GFX908-LABEL: @test_atomicrmw_fadd_bf16_local_strictfp( -; GFX908-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) +; GFX908-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) #[[ATTR6]] ; GFX908-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i32 ; GFX908-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3 ; GFX908-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3 @@ -3101,7 +3101,7 @@ define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf ; GFX908-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]] ; GFX908-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16 ; GFX908-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat -; GFX908-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]] +; GFX908-NEXT: [[NEW:%.*]] = call bfloat @llvm.experimental.constrained.fadd.bf16(bfloat [[TMP4]], bfloat [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR6]] ; GFX908-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16 ; GFX908-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32 ; GFX908-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]] @@ -3118,7 +3118,7 @@ define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf ; GFX908-NEXT: ret bfloat [[TMP7]] ; ; GFX90A-LABEL: @test_atomicrmw_fadd_bf16_local_strictfp( -; GFX90A-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) +; GFX90A-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) #[[ATTR6:[0-9]+]] ; GFX90A-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i32 ; GFX90A-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3 ; GFX90A-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3 @@ -3131,7 +3131,7 @@ define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf ; GFX90A-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]] ; GFX90A-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16 ; GFX90A-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat -; GFX90A-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]] +; GFX90A-NEXT: [[NEW:%.*]] = call bfloat @llvm.experimental.constrained.fadd.bf16(bfloat [[TMP4]], bfloat [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR6]] ; GFX90A-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16 ; GFX90A-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32 ; GFX90A-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]] @@ -3148,7 +3148,7 @@ define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf ; GFX90A-NEXT: ret bfloat [[TMP7]] ; ; GFX940-LABEL: @test_atomicrmw_fadd_bf16_local_strictfp( -; GFX940-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) +; GFX940-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) #[[ATTR6:[0-9]+]] ; GFX940-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i32 ; GFX940-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3 ; GFX940-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3 @@ -3161,7 +3161,7 @@ define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf ; GFX940-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]] ; GFX940-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16 ; GFX940-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat -; GFX940-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]] +; GFX940-NEXT: [[NEW:%.*]] = call bfloat @llvm.experimental.constrained.fadd.bf16(bfloat [[TMP4]], bfloat [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR6]] ; GFX940-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16 ; GFX940-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32 ; GFX940-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]] @@ -3178,7 +3178,7 @@ define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf ; GFX940-NEXT: ret bfloat [[TMP7]] ; ; GFX11-LABEL: @test_atomicrmw_fadd_bf16_local_strictfp( -; GFX11-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) +; GFX11-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) #[[ATTR6]] ; GFX11-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i32 ; GFX11-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3 ; GFX11-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3 @@ -3191,7 +3191,7 @@ define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf ; GFX11-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]] ; GFX11-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16 ; GFX11-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat -; GFX11-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]] +; GFX11-NEXT: [[NEW:%.*]] = call bfloat @llvm.experimental.constrained.fadd.bf16(bfloat [[TMP4]], bfloat [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR6]] ; GFX11-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16 ; GFX11-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32 ; GFX11-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]] -- GitLab From 7daa65a088c00891662d5e7e162e5f8f3283f799 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Fri, 29 Mar 2024 11:58:41 -0700 Subject: [PATCH 0021/1690] Reland "[NVPTX] Use .common linkage for common globals" (#86824) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch from `.weak` to `.common` linkage for common global variables where possible. The `.common` linkage is described in [PTX ISA 11.6.4. Linking Directives: .common] (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#linking-directives-common) > Declares identifier to be globally visible but “common”. > >Common symbols are similar to globally visible symbols. However multiple object files may declare the same common symbol and they may have different types and sizes and references to a symbol get resolved against a common symbol with the largest size. > >Only one object file can initialize a common symbol and that must have the largest size among all other definitions of that common symbol from different object files. > >.common linking directive can be used only on variables with .global storage. It cannot be used on function symbols or on symbols with opaque type. I've updated the logic and tests to only use `.common` for PTX 5.0 or greater and verified that the new tests now pass with `ptxas`. --- llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 16 +++++++------ llvm/test/CodeGen/NVPTX/common-linkage.ll | 29 +++++++++++++++++++++++ llvm/test/CodeGen/NVPTX/weak-global.ll | 9 ++++--- 3 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 llvm/test/CodeGen/NVPTX/common-linkage.ll diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp index ece9821a2d0d..9f31b72bbceb 100644 --- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp @@ -1022,7 +1022,6 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar, const DataLayout &DL = getDataLayout(); // GlobalVariables are always constant pointers themselves. - PointerType *PTy = GVar->getType(); Type *ETy = GVar->getValueType(); if (GVar->hasExternalLinkage()) { @@ -1030,6 +1029,9 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar, O << ".visible "; else O << ".extern "; + } else if (STI.getPTXVersion() >= 50 && GVar->hasCommonLinkage() && + GVar->getAddressSpace() == ADDRESS_SPACE_GLOBAL) { + O << ".common "; } else if (GVar->hasLinkOnceLinkage() || GVar->hasWeakLinkage() || GVar->hasAvailableExternallyLinkage() || GVar->hasCommonLinkage()) { @@ -1141,7 +1143,7 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar, } O << "."; - emitPTXAddressSpace(PTy->getAddressSpace(), O); + emitPTXAddressSpace(GVar->getAddressSpace(), O); if (isManaged(*GVar)) { if (STI.getPTXVersion() < 40 || STI.getSmVersion() < 30) { @@ -1170,8 +1172,8 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar, // Ptx allows variable initilization only for constant and global state // spaces. if (GVar->hasInitializer()) { - if ((PTy->getAddressSpace() == ADDRESS_SPACE_GLOBAL) || - (PTy->getAddressSpace() == ADDRESS_SPACE_CONST)) { + if ((GVar->getAddressSpace() == ADDRESS_SPACE_GLOBAL) || + (GVar->getAddressSpace() == ADDRESS_SPACE_CONST)) { const Constant *Initializer = GVar->getInitializer(); // 'undef' is treated as there is no value specified. if (!Initializer->isNullValue() && !isa(Initializer)) { @@ -1186,7 +1188,7 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar, !isa(GVar->getInitializer())) { report_fatal_error("initial value of '" + GVar->getName() + "' is not allowed in addrspace(" + - Twine(PTy->getAddressSpace()) + ")"); + Twine(GVar->getAddressSpace()) + ")"); } } } @@ -1205,8 +1207,8 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar, ElementSize = DL.getTypeStoreSize(ETy); // Ptx allows variable initilization only for constant and // global state spaces. - if (((PTy->getAddressSpace() == ADDRESS_SPACE_GLOBAL) || - (PTy->getAddressSpace() == ADDRESS_SPACE_CONST)) && + if (((GVar->getAddressSpace() == ADDRESS_SPACE_GLOBAL) || + (GVar->getAddressSpace() == ADDRESS_SPACE_CONST)) && GVar->hasInitializer()) { const Constant *Initializer = GVar->getInitializer(); if (!isa(Initializer) && !Initializer->isNullValue()) { diff --git a/llvm/test/CodeGen/NVPTX/common-linkage.ll b/llvm/test/CodeGen/NVPTX/common-linkage.ll new file mode 100644 index 000000000000..976074e12ba6 --- /dev/null +++ b/llvm/test/CodeGen/NVPTX/common-linkage.ll @@ -0,0 +1,29 @@ +; RUN: llc < %s -march=nvptx -mcpu=sm_20 -mattr=+ptx43 | FileCheck %s --check-prefixes CHECK,PTX43 +; RUN: llc < %s -march=nvptx -mcpu=sm_20 -mattr=+ptx50 | FileCheck %s --check-prefixes CHECK,PTX50 +; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_20 -mattr=+ptx43 | %ptxas-verify %} +; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_20 -mattr=+ptx50 | %ptxas-verify %} + +; PTX43: .weak .global .align 4 .u32 g +; PTX50: .common .global .align 4 .u32 g +@g = common addrspace(1) global i32 0, align 4 + +; CHECK: .weak .const .align 4 .u32 c +@c = common addrspace(4) global i32 0, align 4 + +; CHECK: .weak .shared .align 4 .u32 s +@s = common addrspace(3) global i32 0, align 4 + +define i32 @f1() { + %1 = load i32, ptr addrspace(1) @g + ret i32 %1 +} + +define i32 @f4() { + %1 = load i32, ptr addrspace(4) @c + ret i32 %1 +} + +define i32 @f3() { + %1 = load i32, ptr addrspace(3) @s + ret i32 %1 +} diff --git a/llvm/test/CodeGen/NVPTX/weak-global.ll b/llvm/test/CodeGen/NVPTX/weak-global.ll index dd0160d1c0a6..c5467aad08a3 100644 --- a/llvm/test/CodeGen/NVPTX/weak-global.ll +++ b/llvm/test/CodeGen/NVPTX/weak-global.ll @@ -1,7 +1,10 @@ -; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck %s -; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_20 | %ptxas-verify %} +; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 -mattr=+ptx43 | FileCheck %s --check-prefix PTX43 +; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 -mattr=+ptx50 | FileCheck %s --check-prefix PTX50 +; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_20 -mattr=+ptx43 | %ptxas-verify %} +; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_20 -mattr=+ptx50 | %ptxas-verify %} -; CHECK: .weak .global .align 4 .u32 g +; PTX43: .weak .global .align 4 .u32 g +; PTX50: .common .global .align 4 .u32 g @g = common addrspace(1) global i32 zeroinitializer define i32 @func0() { -- GitLab From ee3a302abaa091e550a80f79694e963d1b5d0b7b Mon Sep 17 00:00:00 2001 From: Jan Svoboda Date: Fri, 29 Mar 2024 12:07:47 -0700 Subject: [PATCH 0022/1690] [clang] Move state out of `PreprocessorOptions` (2/n) (#87099) An instance of `PreprocessorOptions` is part of `CompilerInvocation` which is supposed to be a value type. The `FailedModules` member is problematic, since it's essentially a shared state used by multiple `CompilerInstance` objects, and not really a preprocessor option. Let's move it into `CompilerInstance` instead. --- .../include/clang/Frontend/CompilerInstance.h | 36 +++++++++++++++++++ clang/include/clang/Lex/PreprocessorOptions.h | 22 ------------ clang/lib/Frontend/CompilerInstance.cpp | 27 ++++++-------- 3 files changed, 47 insertions(+), 38 deletions(-) diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index cce91862ae3d..3464654284f1 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -133,6 +133,24 @@ class CompilerInstance : public ModuleLoader { std::vector> DependencyCollectors; + /// Records the set of modules + class FailedModulesSet { + llvm::StringSet<> Failed; + + public: + bool hasAlreadyFailed(StringRef module) { return Failed.count(module) > 0; } + + void addFailed(StringRef module) { Failed.insert(module); } + }; + + /// The set of modules that failed to build. + /// + /// This pointer will be shared among all of the compiler instances created + /// to (re)build modules, so that once a module fails to build anywhere, + /// other instances will see that the module has failed and won't try to + /// build it again. + std::shared_ptr FailedModules; + /// The set of top-level modules that has already been built on the /// fly as part of this overall compilation action. std::map> BuiltModules; @@ -619,6 +637,24 @@ public: } /// @} + /// @name Failed modules set + /// @{ + + bool hasFailedModulesSet() const { return (bool)FailedModules; } + + void createFailedModulesSet() { + FailedModules = std::make_shared(); + } + + std::shared_ptr getFailedModulesSetPtr() const { + return FailedModules; + } + + void setFailedModulesSet(std::shared_ptr FMS) { + FailedModules = FMS; + } + + /// } /// @name Output Files /// @{ diff --git a/clang/include/clang/Lex/PreprocessorOptions.h b/clang/include/clang/Lex/PreprocessorOptions.h index 24e0ca61d414..50b5fba0ff77 100644 --- a/clang/include/clang/Lex/PreprocessorOptions.h +++ b/clang/include/clang/Lex/PreprocessorOptions.h @@ -186,28 +186,6 @@ public: /// with support for lifetime-qualified pointers. ObjCXXARCStandardLibraryKind ObjCXXARCStandardLibrary = ARCXX_nolib; - /// Records the set of modules - class FailedModulesSet { - llvm::StringSet<> Failed; - - public: - bool hasAlreadyFailed(StringRef module) { - return Failed.count(module) > 0; - } - - void addFailed(StringRef module) { - Failed.insert(module); - } - }; - - /// The set of modules that failed to build. - /// - /// This pointer will be shared among all of the compiler instances created - /// to (re)build modules, so that once a module fails to build anywhere, - /// other instances will see that the module has failed and won't try to - /// build it again. - std::shared_ptr FailedModules; - /// Set up preprocessor for RunAnalysis action. bool SetUpStaticAnalyzer = false; diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 79ebb0ae0ee9..6e3baf838644 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -1206,16 +1206,6 @@ compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc, // Note the name of the module we're building. Invocation->getLangOpts().CurrentModule = std::string(ModuleName); - // Make sure that the failed-module structure has been allocated in - // the importing instance, and propagate the pointer to the newly-created - // instance. - PreprocessorOptions &ImportingPPOpts - = ImportingInstance.getInvocation().getPreprocessorOpts(); - if (!ImportingPPOpts.FailedModules) - ImportingPPOpts.FailedModules = - std::make_shared(); - PPOpts.FailedModules = ImportingPPOpts.FailedModules; - // If there is a module map file, build the module using the module map. // Set up the inputs/outputs so that we build the module from its umbrella // header. @@ -1269,6 +1259,13 @@ compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc, SourceMgr.pushModuleBuildStack(ModuleName, FullSourceLoc(ImportLoc, ImportingInstance.getSourceManager())); + // Make sure that the failed-module structure has been allocated in + // the importing instance, and propagate the pointer to the newly-created + // instance. + if (!ImportingInstance.hasFailedModulesSet()) + ImportingInstance.createFailedModulesSet(); + Instance.setFailedModulesSet(ImportingInstance.getFailedModulesSetPtr()); + // If we're collecting module dependencies, we need to share a collector // between all of the module CompilerInstances. Other than that, we don't // want to produce any dependency output from the module build. @@ -1992,10 +1989,8 @@ ModuleLoadResult CompilerInstance::findOrCompileModuleAndReadAST( return nullptr; } - // Check whether we have already attempted to build this module (but - // failed). - if (getPreprocessorOpts().FailedModules && - getPreprocessorOpts().FailedModules->hasAlreadyFailed(ModuleName)) { + // Check whether we have already attempted to build this module (but failed). + if (FailedModules && FailedModules->hasAlreadyFailed(ModuleName)) { getDiagnostics().Report(ModuleNameLoc, diag::err_module_not_built) << ModuleName << SourceRange(ImportLoc, ModuleNameLoc); return nullptr; @@ -2006,8 +2001,8 @@ ModuleLoadResult CompilerInstance::findOrCompileModuleAndReadAST( ModuleFilename)) { assert(getDiagnostics().hasErrorOccurred() && "undiagnosed error in compileModuleAndReadAST"); - if (getPreprocessorOpts().FailedModules) - getPreprocessorOpts().FailedModules->addFailed(ModuleName); + if (FailedModules) + FailedModules->addFailed(ModuleName); return nullptr; } -- GitLab From b42fa8645c4369dfd80397a4e641ec49c96a0d2e Mon Sep 17 00:00:00 2001 From: Helena Kotas Date: Fri, 29 Mar 2024 12:09:44 -0700 Subject: [PATCH 0023/1690] [DXIL] Add lowering for `ceil` (#87043) Add lowering of llvm.ceil intrinsics to DXIL ops. Fixes #86984 --- llvm/lib/Target/DirectX/DXIL.td | 3 +++ llvm/test/CodeGen/DirectX/ceil.ll | 20 ++++++++++++++++++++ llvm/test/CodeGen/DirectX/ceil_error.ll | 10 ++++++++++ 3 files changed, 33 insertions(+) create mode 100644 llvm/test/CodeGen/DirectX/ceil.ll create mode 100644 llvm/test/CodeGen/DirectX/ceil_error.ll diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index a131bcc7dafe..cd388ed3e319 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -292,6 +292,9 @@ def Round : DXILOpMapping<26, unary, int_roundeven, def Floor : DXILOpMapping<27, unary, int_floor, "Returns the largest integer that is less than or equal to the input.", [llvm_halforfloat_ty, LLVMMatchType<0>]>; +def Ceil : DXILOpMapping<28, unary, int_ceil, + "Returns the smallest integer that is greater than or equal to the input.", + [llvm_halforfloat_ty, LLVMMatchType<0>]>; def Trunc : DXILOpMapping<29, unary, int_trunc, "Returns the specified value truncated to the integer component.", [llvm_halforfloat_ty, LLVMMatchType<0>]>; diff --git a/llvm/test/CodeGen/DirectX/ceil.ll b/llvm/test/CodeGen/DirectX/ceil.ll new file mode 100644 index 000000000000..158547146780 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ceil.ll @@ -0,0 +1,20 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for ceil are generated for float and half. + +define noundef float @ceil_float(float noundef %a) { +entry: +; CHECK:call float @dx.op.unary.f32(i32 28, float %{{.*}}) + %elt.ceil = call float @llvm.ceil.f32(float %a) + ret float %elt.ceil +} + +define noundef half @ceil_half(half noundef %a) { +entry: +; CHECK:call half @dx.op.unary.f16(i32 28, half %{{.*}}) + %elt.ceil = call half @llvm.ceil.f16(half %a) + ret half %elt.ceil +} + +declare half @llvm.ceil.f16(half) +declare float @llvm.ceil.f32(float) diff --git a/llvm/test/CodeGen/DirectX/ceil_error.ll b/llvm/test/CodeGen/DirectX/ceil_error.ll new file mode 100644 index 000000000000..1b554d871556 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ceil_error.ll @@ -0,0 +1,10 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; DXIL operation ceil does not support double overload type +; CHECK: LLVM ERROR: Invalid Overload Type + +define noundef double @ceil_double(double noundef %a) { +entry: + %elt.ceil = call double @llvm.ceil.f64(double %a) + ret double %elt.ceil +} -- GitLab From 360f7f5674c4ddef006d233ef7259f8fe07ae81b Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Fri, 29 Mar 2024 15:27:28 -0400 Subject: [PATCH 0024/1690] [GlobalISel] Call `setInstrAndDebugLoc` before `tryCombineAll` (#86993) This can remove all unnecessary redundant calls in each combiner. --- .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 62 ++----------------- .../GlobalISelCombinerEmitter/match-table.td | 3 + .../TableGen/GlobalISelCombinerEmitter.cpp | 1 + 3 files changed, 9 insertions(+), 57 deletions(-) diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index f3a9ade51559..98e7c73a801f 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -872,7 +872,6 @@ bool CombinerHelper::matchSextTruncSextLoad(MachineInstr &MI) { void CombinerHelper::applySextTruncSextLoad(MachineInstr &MI) { assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG); - Builder.setInstrAndDebugLoc(MI); Builder.buildCopy(MI.getOperand(0).getReg(), MI.getOperand(1).getReg()); MI.eraseFromParent(); } @@ -1299,7 +1298,6 @@ bool CombinerHelper::matchCombineIndexedLoadStore( void CombinerHelper::applyCombineIndexedLoadStore( MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) { MachineInstr &AddrDef = *MRI.getUniqueVRegDef(MatchInfo.Addr); - Builder.setInstrAndDebugLoc(MI); unsigned Opcode = MI.getOpcode(); bool IsStore = Opcode == TargetOpcode::G_STORE; unsigned NewOpcode = getIndexedOpc(Opcode); @@ -1416,14 +1414,8 @@ void CombinerHelper::applyCombineDivRem(MachineInstr &MI, // deps by "moving" the instruction incorrectly. Also keep track of which // instruction is first so we pick it's operands, avoiding use-before-def // bugs. - MachineInstr *FirstInst; - if (dominates(MI, *OtherMI)) { - Builder.setInstrAndDebugLoc(MI); - FirstInst = &MI; - } else { - Builder.setInstrAndDebugLoc(*OtherMI); - FirstInst = OtherMI; - } + MachineInstr *FirstInst = dominates(MI, *OtherMI) ? &MI : OtherMI; + Builder.setInstrAndDebugLoc(*FirstInst); Builder.buildInstr(IsSigned ? TargetOpcode::G_SDIVREM : TargetOpcode::G_UDIVREM, @@ -1556,7 +1548,6 @@ static APFloat constantFoldFpUnary(const MachineInstr &MI, void CombinerHelper::applyCombineConstantFoldFpUnary(MachineInstr &MI, const ConstantFP *Cst) { - Builder.setInstrAndDebugLoc(MI); APFloat Folded = constantFoldFpUnary(MI, MRI, Cst->getValue()); const ConstantFP *NewCst = ConstantFP::get(Builder.getContext(), Folded); Builder.buildFConstant(MI.getOperand(0), *NewCst); @@ -1691,7 +1682,6 @@ void CombinerHelper::applyShiftImmedChain(MachineInstr &MI, Opcode == TargetOpcode::G_USHLSAT) && "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT"); - Builder.setInstrAndDebugLoc(MI); LLT Ty = MRI.getType(MI.getOperand(1).getReg()); unsigned const ScalarSizeInBits = Ty.getScalarSizeInBits(); auto Imm = MatchInfo.Imm; @@ -1807,7 +1797,6 @@ void CombinerHelper::applyShiftOfShiftedLogic(MachineInstr &MI, LLT ShlType = MRI.getType(MI.getOperand(2).getReg()); LLT DestType = MRI.getType(MI.getOperand(0).getReg()); - Builder.setInstrAndDebugLoc(MI); Register Const = Builder.buildConstant(ShlType, MatchInfo.ValSum).getReg(0); @@ -1943,7 +1932,6 @@ void CombinerHelper::applyCombineShlOfExtend(MachineInstr &MI, int64_t ShiftAmtVal = MatchData.Imm; LLT ExtSrcTy = MRI.getType(ExtSrcReg); - Builder.setInstrAndDebugLoc(MI); auto ShiftAmt = Builder.buildConstant(ExtSrcTy, ShiftAmtVal); auto NarrowShift = Builder.buildShl(ExtSrcTy, ExtSrcReg, ShiftAmt, MI.getFlags()); @@ -2013,7 +2001,6 @@ void CombinerHelper::applyCombineUnmergeMergeToPlainValues( LLT SrcTy = MRI.getType(Operands[0]); LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); bool CanReuseInputDirectly = DstTy == SrcTy; - Builder.setInstrAndDebugLoc(MI); for (unsigned Idx = 0; Idx < NumElems; ++Idx) { Register DstReg = MI.getOperand(Idx).getReg(); Register SrcReg = Operands[Idx]; @@ -2066,7 +2053,6 @@ void CombinerHelper::applyCombineUnmergeConstant(MachineInstr &MI, assert((MI.getNumOperands() - 1 == Csts.size()) && "Not enough operands to replace all defs"); unsigned NumElems = MI.getNumOperands() - 1; - Builder.setInstrAndDebugLoc(MI); for (unsigned Idx = 0; Idx < NumElems; ++Idx) { Register DstReg = MI.getOperand(Idx).getReg(); Builder.buildConstant(DstReg, Csts[Idx]); @@ -2104,7 +2090,6 @@ bool CombinerHelper::matchCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI) { } void CombinerHelper::applyCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI) { - Builder.setInstrAndDebugLoc(MI); Register SrcReg = MI.getOperand(MI.getNumDefs()).getReg(); Register Dst0Reg = MI.getOperand(0).getReg(); Builder.buildTrunc(Dst0Reg, SrcReg); @@ -2152,8 +2137,6 @@ void CombinerHelper::applyCombineUnmergeZExtToZExt(MachineInstr &MI) { LLT Dst0Ty = MRI.getType(Dst0Reg); LLT ZExtSrcTy = MRI.getType(ZExtSrcReg); - Builder.setInstrAndDebugLoc(MI); - if (Dst0Ty.getSizeInBits() > ZExtSrcTy.getSizeInBits()) { Builder.buildZExt(Dst0Reg, ZExtSrcReg); } else { @@ -2207,7 +2190,6 @@ void CombinerHelper::applyCombineShiftToUnmerge(MachineInstr &MI, LLT HalfTy = LLT::scalar(HalfSize); - Builder.setInstr(MI); auto Unmerge = Builder.buildUnmerge(HalfTy, SrcReg); unsigned NarrowShiftAmt = ShiftVal - HalfSize; @@ -2292,7 +2274,6 @@ bool CombinerHelper::matchCombineI2PToP2I(MachineInstr &MI, Register &Reg) { void CombinerHelper::applyCombineI2PToP2I(MachineInstr &MI, Register &Reg) { assert(MI.getOpcode() == TargetOpcode::G_INTTOPTR && "Expected a G_INTTOPTR"); Register DstReg = MI.getOperand(0).getReg(); - Builder.setInstr(MI); Builder.buildCopy(DstReg, Reg); MI.eraseFromParent(); } @@ -2300,7 +2281,6 @@ void CombinerHelper::applyCombineI2PToP2I(MachineInstr &MI, Register &Reg) { void CombinerHelper::applyCombineP2IToI2P(MachineInstr &MI, Register &Reg) { assert(MI.getOpcode() == TargetOpcode::G_PTRTOINT && "Expected a G_PTRTOINT"); Register DstReg = MI.getOperand(0).getReg(); - Builder.setInstr(MI); Builder.buildZExtOrTrunc(DstReg, Reg); MI.eraseFromParent(); } @@ -2343,7 +2323,6 @@ void CombinerHelper::applyCombineAddP2IToPtrAdd( LLT PtrTy = MRI.getType(LHS); - Builder.setInstrAndDebugLoc(MI); auto PtrAdd = Builder.buildPtrAdd(PtrTy, LHS, RHS); Builder.buildPtrToInt(Dst, PtrAdd); MI.eraseFromParent(); @@ -2375,7 +2354,6 @@ void CombinerHelper::applyCombineConstPtrAddToI2P(MachineInstr &MI, auto &PtrAdd = cast(MI); Register Dst = PtrAdd.getReg(0); - Builder.setInstrAndDebugLoc(MI); Builder.buildConstant(Dst, NewCst); PtrAdd.eraseFromParent(); } @@ -2455,7 +2433,6 @@ void CombinerHelper::applyCombineExtOfExt( (MI.getOpcode() == TargetOpcode::G_SEXT && SrcExtOp == TargetOpcode::G_ZEXT)) { Register DstReg = MI.getOperand(0).getReg(); - Builder.setInstrAndDebugLoc(MI); Builder.buildInstr(SrcExtOp, {DstReg}, {Reg}); MI.eraseFromParent(); } @@ -2488,7 +2465,6 @@ void CombinerHelper::applyCombineTruncOfExt( replaceRegWith(MRI, DstReg, SrcReg); return; } - Builder.setInstrAndDebugLoc(MI); if (SrcTy.getSizeInBits() < DstTy.getSizeInBits()) Builder.buildInstr(SrcExtOp, {DstReg}, {SrcReg}); else @@ -2576,8 +2552,6 @@ bool CombinerHelper::matchCombineTruncOfShift( void CombinerHelper::applyCombineTruncOfShift( MachineInstr &MI, std::pair &MatchInfo) { - Builder.setInstrAndDebugLoc(MI); - MachineInstr *ShiftMI = MatchInfo.first; LLT NewShiftTy = MatchInfo.second; @@ -2823,7 +2797,6 @@ void CombinerHelper::applyFunnelShiftConstantModulo(MachineInstr &MI) { APInt NewConst = VRegAndVal->Value.urem( APInt(ConstTy.getSizeInBits(), DstTy.getScalarSizeInBits())); - Builder.setInstrAndDebugLoc(MI); auto NewConstInstr = Builder.buildConstant(ConstTy, NewConst.getZExtValue()); Builder.buildInstr( MI.getOpcode(), {MI.getOperand(0)}, @@ -2866,35 +2839,31 @@ bool CombinerHelper::matchOperandIsKnownToBeAPowerOfTwo(MachineInstr &MI, void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI, double C) { assert(MI.getNumDefs() == 1 && "Expected only one def?"); - Builder.setInstr(MI); Builder.buildFConstant(MI.getOperand(0), C); MI.eraseFromParent(); } void CombinerHelper::replaceInstWithConstant(MachineInstr &MI, int64_t C) { assert(MI.getNumDefs() == 1 && "Expected only one def?"); - Builder.setInstr(MI); Builder.buildConstant(MI.getOperand(0), C); MI.eraseFromParent(); } void CombinerHelper::replaceInstWithConstant(MachineInstr &MI, APInt C) { assert(MI.getNumDefs() == 1 && "Expected only one def?"); - Builder.setInstr(MI); Builder.buildConstant(MI.getOperand(0), C); MI.eraseFromParent(); } -void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI, ConstantFP *CFP) { +void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI, + ConstantFP *CFP) { assert(MI.getNumDefs() == 1 && "Expected only one def?"); - Builder.setInstr(MI); Builder.buildFConstant(MI.getOperand(0), CFP->getValueAPF()); MI.eraseFromParent(); } void CombinerHelper::replaceInstWithUndef(MachineInstr &MI) { assert(MI.getNumDefs() == 1 && "Expected only one def?"); - Builder.setInstr(MI); Builder.buildUndef(MI.getOperand(0)); MI.eraseFromParent(); } @@ -2962,7 +2931,6 @@ bool CombinerHelper::matchCombineInsertVecElts( void CombinerHelper::applyCombineInsertVecElts( MachineInstr &MI, SmallVectorImpl &MatchInfo) { - Builder.setInstr(MI); Register UndefReg; auto GetUndef = [&]() { if (UndefReg) @@ -2981,7 +2949,6 @@ void CombinerHelper::applyCombineInsertVecElts( void CombinerHelper::applySimplifyAddToSub( MachineInstr &MI, std::tuple &MatchInfo) { - Builder.setInstr(MI); Register SubLHS, SubRHS; std::tie(SubLHS, SubRHS) = MatchInfo; Builder.buildSub(MI.getOperand(0).getReg(), SubLHS, SubRHS); @@ -3084,7 +3051,6 @@ void CombinerHelper::applyBuildInstructionSteps( MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) { assert(MatchInfo.InstrsToBuild.size() && "Expected at least one instr to build?"); - Builder.setInstr(MI); for (auto &InstrToBuild : MatchInfo.InstrsToBuild) { assert(InstrToBuild.Opcode && "Expected a valid opcode?"); assert(InstrToBuild.OperandFns.size() && "Expected at least one operand?"); @@ -3120,7 +3086,6 @@ void CombinerHelper::applyAshShlToSextInreg( int64_t ShiftAmt; std::tie(Src, ShiftAmt) = MatchInfo; unsigned Size = MRI.getType(Src).getScalarSizeInBits(); - Builder.setInstrAndDebugLoc(MI); Builder.buildSExtInReg(MI.getOperand(0).getReg(), Src, Size - ShiftAmt); MI.eraseFromParent(); } @@ -3399,7 +3364,6 @@ bool CombinerHelper::matchXorOfAndWithSameReg( void CombinerHelper::applyXorOfAndWithSameReg( MachineInstr &MI, std::pair &MatchInfo) { // Fold (xor (and x, y), y) -> (and (not x), y) - Builder.setInstrAndDebugLoc(MI); Register X, Y; std::tie(X, Y) = MatchInfo; auto Not = Builder.buildNot(MRI.getType(X), X); @@ -3431,7 +3395,6 @@ bool CombinerHelper::matchPtrAddZero(MachineInstr &MI) { void CombinerHelper::applyPtrAddZero(MachineInstr &MI) { auto &PtrAdd = cast(MI); - Builder.setInstrAndDebugLoc(PtrAdd); Builder.buildIntToPtr(PtrAdd.getReg(0), PtrAdd.getOffsetReg()); PtrAdd.eraseFromParent(); } @@ -3442,7 +3405,6 @@ void CombinerHelper::applySimplifyURemByPow2(MachineInstr &MI) { Register Src0 = MI.getOperand(1).getReg(); Register Pow2Src1 = MI.getOperand(2).getReg(); LLT Ty = MRI.getType(DstReg); - Builder.setInstrAndDebugLoc(MI); // Fold (urem x, pow2) -> (and x, pow2-1) auto NegOne = Builder.buildConstant(Ty, -1); @@ -3507,8 +3469,6 @@ bool CombinerHelper::matchFoldBinOpIntoSelect(MachineInstr &MI, /// to fold. void CombinerHelper::applyFoldBinOpIntoSelect(MachineInstr &MI, const unsigned &SelectOperand) { - Builder.setInstrAndDebugLoc(MI); - Register Dst = MI.getOperand(0).getReg(); Register LHS = MI.getOperand(1).getReg(); Register RHS = MI.getOperand(2).getReg(); @@ -4029,7 +3989,6 @@ void CombinerHelper::applyExtractVecEltBuildVec(MachineInstr &MI, Register DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); - Builder.setInstrAndDebugLoc(MI); if (ScalarTy != DstTy) { assert(ScalarTy.getSizeInBits() > DstTy.getSizeInBits()); Builder.buildTrunc(DstReg, Reg); @@ -4095,14 +4054,12 @@ void CombinerHelper::applyExtractAllEltsFromBuildVector( void CombinerHelper::applyBuildFn( MachineInstr &MI, std::function &MatchInfo) { - Builder.setInstrAndDebugLoc(MI); - MatchInfo(Builder); + applyBuildFnNoErase(MI, MatchInfo); MI.eraseFromParent(); } void CombinerHelper::applyBuildFnNoErase( MachineInstr &MI, std::function &MatchInfo) { - Builder.setInstrAndDebugLoc(MI); MatchInfo(Builder); } @@ -4204,7 +4161,6 @@ void CombinerHelper::applyRotateOutOfRange(MachineInstr &MI) { MI.getOpcode() == TargetOpcode::G_ROTR); unsigned Bitsize = MRI.getType(MI.getOperand(0).getReg()).getScalarSizeInBits(); - Builder.setInstrAndDebugLoc(MI); Register Amt = MI.getOperand(2).getReg(); LLT AmtTy = MRI.getType(Amt); auto Bits = Builder.buildConstant(AmtTy, Bitsize); @@ -5027,7 +4983,6 @@ MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) { LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(Ty); LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType(); auto &MIB = Builder; - MIB.setInstrAndDebugLoc(MI); bool UseNPQ = false; SmallVector PreShifts, PostShifts, MagicFactors, NPQFactors; @@ -5213,7 +5168,6 @@ MachineInstr *CombinerHelper::buildSDivUsingMul(MachineInstr &MI) { LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(Ty); LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType(); auto &MIB = Builder; - MIB.setInstrAndDebugLoc(MI); bool UseSRA = false; SmallVector Shifts, Factors; @@ -5295,8 +5249,6 @@ void CombinerHelper::applySDivByPow2(MachineInstr &MI) { LLT CCVT = Ty.isVector() ? LLT::vector(Ty.getElementCount(), 1) : LLT::scalar(1); - Builder.setInstrAndDebugLoc(MI); - // Effectively we want to lower G_SDIV %lhs, %rhs, where %rhs is a power of 2, // to the following version: // @@ -5354,8 +5306,6 @@ void CombinerHelper::applyUDivByPow2(MachineInstr &MI) { LLT Ty = MRI.getType(Dst); LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(Ty); - Builder.setInstrAndDebugLoc(MI); - auto C1 = Builder.buildCTTZ(ShiftAmtTy, RHS); Builder.buildLShr(MI.getOperand(0).getReg(), LHS, C1); MI.eraseFromParent(); @@ -5385,7 +5335,6 @@ void CombinerHelper::applyUMulHToLShr(MachineInstr &MI) { LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(Ty); unsigned NumEltBits = Ty.getScalarSizeInBits(); - Builder.setInstrAndDebugLoc(MI); auto LogBase2 = buildLogBase2(RHS, Builder); auto ShiftAmt = Builder.buildSub(Ty, Builder.buildConstant(Ty, NumEltBits), LogBase2); @@ -5465,7 +5414,6 @@ bool CombinerHelper::matchFsubToFneg(MachineInstr &MI, Register &MatchInfo) { } void CombinerHelper::applyFsubToFneg(MachineInstr &MI, Register &MatchInfo) { - Builder.setInstrAndDebugLoc(MI); Register Dst = MI.getOperand(0).getReg(); Builder.buildFNeg( Dst, Builder.buildFCanonicalize(MRI.getType(Dst), MatchInfo).getReg(0)); diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td index 5cf4e044a0fb..0189d3d056fc 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td @@ -114,14 +114,17 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: void GenMyCombiner::runCustomAction(unsigned ApplyID, const MatcherState &State, NewMIVector &OutMIs) const { // CHECK-NEXT: switch(ApplyID) { // CHECK-NEXT: case GICXXCustomAction_CombineApplyGICombiner0:{ +// CHECK-NEXT: Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]); // CHECK-NEXT: APPLY // CHECK-NEXT: return; // CHECK-NEXT: } // CHECK-NEXT: case GICXXCustomAction_CombineApplyGICombiner1:{ +// CHECK-NEXT: Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]); // CHECK-NEXT: APPLY MatchInfos.MDInfo0, MatchInfos.MDInfo1 // CHECK-NEXT: return; // CHECK-NEXT: } // CHECK-NEXT: case GICXXCustomAction_CombineApplyGICombiner2:{ +// CHECK-NEXT: Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]); // CHECK-NEXT: APPLY State.MIs[1]->getOperand(1) State.MIs[0]->getOperand(1) OutMIs[0] // CHECK-NEXT: return; // CHECK-NEXT: } diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp index 1ae6efd4a7d0..ef9e9ff04f85 100644 --- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp @@ -2459,6 +2459,7 @@ void GICombinerEmitter::emitRunCustomAction(raw_ostream &OS) { OS << " switch(ApplyID) {\n"; for (const auto &Apply : ApplyCode) { OS << " case " << Apply->getEnumNameWithPrefix(CXXApplyPrefix) << ":{\n" + << " Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]);\n" << " " << join(split(Apply->Code, '\n'), "\n ") << '\n' << " return;\n"; OS << " }\n"; -- GitLab From 3a106e5b2cd9f4073b2961b991ebaeee96786309 Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Fri, 29 Mar 2024 15:59:50 -0400 Subject: [PATCH 0025/1690] [GlobalISel] Fold G_ICMP if possible (#86357) This patch tries to fold `G_ICMP` if possible. --- llvm/include/llvm/CodeGen/GlobalISel/Utils.h | 4 + llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp | 14 + .../CodeGen/GlobalISel/LegalizerHelper.cpp | 34 ++- llvm/lib/CodeGen/GlobalISel/Utils.cpp | 68 +++++ .../AArch64/GlobalISel/arm64-atomic.ll | 72 ++--- .../AArch64/GlobalISel/combine-udiv.ll | 92 +++---- .../AArch64/GlobalISel/combine-udiv.mir | 245 ++++++++---------- .../GlobalISel/legalize-cmpxchg-128.mir | 29 +-- .../legalize-cmpxchg-with-success.mir | 36 +-- .../legalize-atomic-cmpxchg-with-success.mir | 15 +- .../AMDGPU/GlobalISel/legalize-saddo.mir | 15 +- .../AMDGPU/GlobalISel/legalize-saddsat.mir | 45 ++-- .../AMDGPU/GlobalISel/legalize-ssubo.mir | 15 +- .../AMDGPU/GlobalISel/legalize-ssubsat.mir | 45 ++-- .../CodeGen/Mips/GlobalISel/legalizer/add.mir | 21 +- .../Mips/GlobalISel/legalizer/cttz.mir | 4 +- .../CodeGen/Mips/GlobalISel/legalizer/mul.mir | 34 ++- .../legalizer/legalize-add-rv32.mir | 21 +- .../legalizer/legalize-add-rv64.mir | 21 +- .../legalizer/legalize-addo-subo-rv32.mir | 33 ++- .../legalizer/legalize-addo-subo-rv64.mir | 12 +- .../legalizer/legalize-mul-ext-rv32.mir | 21 +- .../legalizer/legalize-mul-ext-rv64.mir | 21 +- llvm/unittests/CodeGen/GlobalISel/CSETest.cpp | 170 ++++++++++++ 24 files changed, 694 insertions(+), 393 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h index 0241ec4f2111..807cec3c177d 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h @@ -315,6 +315,10 @@ std::optional> ConstantFoldCountZeros(Register Src, const MachineRegisterInfo &MRI, std::function CB); +std::optional> +ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2, + const MachineRegisterInfo &MRI); + /// Test if the given value is known to have exactly one bit set. This differs /// from computeKnownBits in that it doesn't necessarily determine which bit is /// set. diff --git a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp index a0bc325c6cda..551ba1e6036c 100644 --- a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp @@ -174,6 +174,20 @@ MachineInstrBuilder CSEMIRBuilder::buildInstr(unsigned Opc, switch (Opc) { default: break; + case TargetOpcode::G_ICMP: { + assert(SrcOps.size() == 3 && "Invalid sources"); + assert(DstOps.size() == 1 && "Invalid dsts"); + LLT SrcTy = SrcOps[1].getLLTTy(*getMRI()); + + if (std::optional> Cst = + ConstantFoldICmp(SrcOps[0].getPredicate(), SrcOps[1].getReg(), + SrcOps[2].getReg(), *getMRI())) { + if (SrcTy.isVector()) + return buildBuildVectorConstant(DstOps[0], *Cst); + return buildConstant(DstOps[0], Cst->front()); + } + break; + } case TargetOpcode::G_ADD: case TargetOpcode::G_PTR_ADD: case TargetOpcode::G_AND: diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index c00fe1f8ff54..797bbf7efe60 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -3768,9 +3768,11 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) { } case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { auto [OldValRes, SuccessRes, Addr, CmpVal, NewVal] = MI.getFirst5Regs(); - MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal, + Register NewOldValRes = MRI.cloneVirtualRegister(OldValRes); + MIRBuilder.buildAtomicCmpXchg(NewOldValRes, Addr, CmpVal, NewVal, **MI.memoperands_begin()); - MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal); + MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, NewOldValRes, CmpVal); + MIRBuilder.buildCopy(OldValRes, NewOldValRes); MI.eraseFromParent(); return Legalized; } @@ -3789,8 +3791,12 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) { case G_UADDO: { auto [Res, CarryOut, LHS, RHS] = MI.getFirst4Regs(); - MIRBuilder.buildAdd(Res, LHS, RHS); - MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS); + Register NewRes = MRI.cloneVirtualRegister(Res); + + MIRBuilder.buildAdd(NewRes, LHS, RHS); + MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, NewRes, RHS); + + MIRBuilder.buildCopy(Res, NewRes); MI.eraseFromParent(); return Legalized; @@ -3800,6 +3806,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) { const LLT CondTy = MRI.getType(CarryOut); const LLT Ty = MRI.getType(Res); + Register NewRes = MRI.cloneVirtualRegister(Res); + // Initial add of the two operands. auto TmpRes = MIRBuilder.buildAdd(Ty, LHS, RHS); @@ -3808,15 +3816,18 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) { // Add the sum and the carry. auto ZExtCarryIn = MIRBuilder.buildZExt(Ty, CarryIn); - MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn); + MIRBuilder.buildAdd(NewRes, TmpRes, ZExtCarryIn); // Second check for carry. We can only carry if the initial sum is all 1s // and the carry is set, resulting in a new sum of 0. auto Zero = MIRBuilder.buildConstant(Ty, 0); - auto ResEqZero = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, Res, Zero); + auto ResEqZero = + MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, NewRes, Zero); auto Carry2 = MIRBuilder.buildAnd(CondTy, ResEqZero, CarryIn); MIRBuilder.buildOr(CarryOut, Carry, Carry2); + MIRBuilder.buildCopy(Res, NewRes); + MI.eraseFromParent(); return Legalized; } @@ -7671,10 +7682,12 @@ LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) { LLT Ty = Dst0Ty; LLT BoolTy = Dst1Ty; + Register NewDst0 = MRI.cloneVirtualRegister(Dst0); + if (IsAdd) - MIRBuilder.buildAdd(Dst0, LHS, RHS); + MIRBuilder.buildAdd(NewDst0, LHS, RHS); else - MIRBuilder.buildSub(Dst0, LHS, RHS); + MIRBuilder.buildSub(NewDst0, LHS, RHS); // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow. @@ -7687,12 +7700,15 @@ LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) { // (LHS) if and only if the other operand (RHS) is (non-zero) positive, // otherwise there will be overflow. auto ResultLowerThanLHS = - MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, Dst0, LHS); + MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, NewDst0, LHS); auto ConditionRHS = MIRBuilder.buildICmp( IsAdd ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGT, BoolTy, RHS, Zero); MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS); + + MIRBuilder.buildCopy(Dst0, NewDst0); MI.eraseFromParent(); + return Legalized; } diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp index 8c41f8b1bdcd..c3bc3203b636 100644 --- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp @@ -997,6 +997,74 @@ llvm::ConstantFoldCountZeros(Register Src, const MachineRegisterInfo &MRI, return std::nullopt; } +std::optional> +llvm::ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2, + const MachineRegisterInfo &MRI) { + LLT Ty = MRI.getType(Op1); + if (Ty != MRI.getType(Op2)) + return std::nullopt; + + auto TryFoldScalar = [&MRI, Pred](Register LHS, + Register RHS) -> std::optional { + auto LHSCst = getIConstantVRegVal(LHS, MRI); + auto RHSCst = getIConstantVRegVal(RHS, MRI); + if (!LHSCst || !RHSCst) + return std::nullopt; + + switch (Pred) { + case CmpInst::Predicate::ICMP_EQ: + return APInt(/*numBits=*/1, LHSCst->eq(*RHSCst)); + case CmpInst::Predicate::ICMP_NE: + return APInt(/*numBits=*/1, LHSCst->ne(*RHSCst)); + case CmpInst::Predicate::ICMP_UGT: + return APInt(/*numBits=*/1, LHSCst->ugt(*RHSCst)); + case CmpInst::Predicate::ICMP_UGE: + return APInt(/*numBits=*/1, LHSCst->uge(*RHSCst)); + case CmpInst::Predicate::ICMP_ULT: + return APInt(/*numBits=*/1, LHSCst->ult(*RHSCst)); + case CmpInst::Predicate::ICMP_ULE: + return APInt(/*numBits=*/1, LHSCst->ule(*RHSCst)); + case CmpInst::Predicate::ICMP_SGT: + return APInt(/*numBits=*/1, LHSCst->sgt(*RHSCst)); + case CmpInst::Predicate::ICMP_SGE: + return APInt(/*numBits=*/1, LHSCst->sge(*RHSCst)); + case CmpInst::Predicate::ICMP_SLT: + return APInt(/*numBits=*/1, LHSCst->slt(*RHSCst)); + case CmpInst::Predicate::ICMP_SLE: + return APInt(/*numBits=*/1, LHSCst->sle(*RHSCst)); + default: + return std::nullopt; + } + }; + + SmallVector FoldedICmps; + + if (Ty.isVector()) { + // Try to constant fold each element. + auto *BV1 = getOpcodeDef(Op1, MRI); + auto *BV2 = getOpcodeDef(Op2, MRI); + if (!BV1 || !BV2) + return std::nullopt; + assert(BV1->getNumSources() == BV2->getNumSources() && "Invalid vectors"); + for (unsigned I = 0; I < BV1->getNumSources(); ++I) { + if (auto MaybeFold = + TryFoldScalar(BV1->getSourceReg(I), BV2->getSourceReg(I))) { + FoldedICmps.emplace_back(*MaybeFold); + continue; + } + return std::nullopt; + } + return FoldedICmps; + } + + if (auto MaybeCst = TryFoldScalar(Op1, Op2)) { + FoldedICmps.emplace_back(*MaybeCst); + return FoldedICmps; + } + + return std::nullopt; +} + bool llvm::isKnownToBeAPowerOfTwo(Register Reg, const MachineRegisterInfo &MRI, GISelKnownBits *KB) { std::optional DefSrcReg = diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll index 458c2cb76d9e..7163da0dc024 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll @@ -512,9 +512,9 @@ define i32 @fetch_and_nand(ptr %p) #0 { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB6_2 ; CHECK-NOLSE-O0-NEXT: LBB6_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB6_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB6_1 ; CHECK-NOLSE-O0-NEXT: b LBB6_5 @@ -540,9 +540,9 @@ define i32 @fetch_and_nand(ptr %p) #0 { ; CHECK-OUTLINE-O0-NEXT: mvn w1, w8 ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas4_rel ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #8] ; 4-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: subs w8, w0, w8 ; CHECK-OUTLINE-O0-NEXT: cset w8, eq +; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: tbz w8, #0, LBB6_1 ; CHECK-OUTLINE-O0-NEXT: b LBB6_2 @@ -582,9 +582,9 @@ define i32 @fetch_and_nand(ptr %p) #0 { ; CHECK-LSE-O0-NEXT: mvn w10, w9 ; CHECK-LSE-O0-NEXT: mov x9, x8 ; CHECK-LSE-O0-NEXT: casl w9, w10, [x11] -; CHECK-LSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-LSE-O0-NEXT: subs w8, w9, w8 ; CHECK-LSE-O0-NEXT: cset w8, eq +; CHECK-LSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-LSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-LSE-O0-NEXT: tbz w8, #0, LBB6_1 ; CHECK-LSE-O0-NEXT: b LBB6_2 @@ -649,9 +649,9 @@ define i64 @fetch_and_nand_64(ptr %p) #0 { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB7_2 ; CHECK-NOLSE-O0-NEXT: LBB7_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB7_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB7_1 ; CHECK-NOLSE-O0-NEXT: b LBB7_5 @@ -677,9 +677,9 @@ define i64 @fetch_and_nand_64(ptr %p) #0 { ; CHECK-OUTLINE-O0-NEXT: mvn x1, x8 ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas8_acq_rel ; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp] ; 8-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #8] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: subs x8, x0, x8 ; CHECK-OUTLINE-O0-NEXT: cset w8, eq +; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #8] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #24] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: tbz w8, #0, LBB7_1 ; CHECK-OUTLINE-O0-NEXT: b LBB7_2 @@ -719,9 +719,9 @@ define i64 @fetch_and_nand_64(ptr %p) #0 { ; CHECK-LSE-O0-NEXT: mvn x10, x9 ; CHECK-LSE-O0-NEXT: mov x9, x8 ; CHECK-LSE-O0-NEXT: casal x9, x10, [x11] -; CHECK-LSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill ; CHECK-LSE-O0-NEXT: subs x8, x9, x8 ; CHECK-LSE-O0-NEXT: cset w8, eq +; CHECK-LSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill ; CHECK-LSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-LSE-O0-NEXT: tbz w8, #0, LBB7_1 ; CHECK-LSE-O0-NEXT: b LBB7_2 @@ -782,9 +782,9 @@ define i32 @fetch_and_or(ptr %p) #0 { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB8_2 ; CHECK-NOLSE-O0-NEXT: LBB8_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB8_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB8_1 ; CHECK-NOLSE-O0-NEXT: b LBB8_5 @@ -855,9 +855,9 @@ define i64 @fetch_and_or_64(ptr %p) #0 { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB9_2 ; CHECK-NOLSE-O0-NEXT: LBB9_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB9_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB9_1 ; CHECK-NOLSE-O0-NEXT: b LBB9_5 @@ -4005,9 +4005,9 @@ define i32 @atomicrmw_add_i32(ptr %ptr, i32 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB47_2 ; CHECK-NOLSE-O0-NEXT: LBB47_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB47_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB47_1 ; CHECK-NOLSE-O0-NEXT: b LBB47_5 @@ -4097,9 +4097,9 @@ define i32 @atomicrmw_xchg_i32(ptr %ptr, i32 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB48_2 ; CHECK-NOLSE-O0-NEXT: LBB48_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB48_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB48_1 ; CHECK-NOLSE-O0-NEXT: b LBB48_5 @@ -4190,9 +4190,9 @@ define i32 @atomicrmw_sub_i32(ptr %ptr, i32 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB49_2 ; CHECK-NOLSE-O0-NEXT: LBB49_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB49_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB49_1 ; CHECK-NOLSE-O0-NEXT: b LBB49_5 @@ -4287,9 +4287,9 @@ define i32 @atomicrmw_and_i32(ptr %ptr, i32 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB50_2 ; CHECK-NOLSE-O0-NEXT: LBB50_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB50_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB50_1 ; CHECK-NOLSE-O0-NEXT: b LBB50_5 @@ -4384,9 +4384,9 @@ define i32 @atomicrmw_or_i32(ptr %ptr, i32 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB51_2 ; CHECK-NOLSE-O0-NEXT: LBB51_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB51_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB51_1 ; CHECK-NOLSE-O0-NEXT: b LBB51_5 @@ -4477,9 +4477,9 @@ define i32 @atomicrmw_xor_i32(ptr %ptr, i32 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB52_2 ; CHECK-NOLSE-O0-NEXT: LBB52_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB52_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB52_1 ; CHECK-NOLSE-O0-NEXT: b LBB52_5 @@ -4572,9 +4572,9 @@ define i32 @atomicrmw_min_i32(ptr %ptr, i32 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB53_2 ; CHECK-NOLSE-O0-NEXT: LBB53_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB53_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB53_1 ; CHECK-NOLSE-O0-NEXT: b LBB53_5 @@ -4605,9 +4605,9 @@ define i32 @atomicrmw_min_i32(ptr %ptr, i32 %rhs) { ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, le ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas4_acq ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #8] ; 4-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: subs w8, w0, w8 ; CHECK-OUTLINE-O0-NEXT: cset w8, eq +; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: tbz w8, #0, LBB53_1 ; CHECK-OUTLINE-O0-NEXT: b LBB53_2 @@ -4686,9 +4686,9 @@ define i32 @atomicrmw_max_i32(ptr %ptr, i32 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB54_2 ; CHECK-NOLSE-O0-NEXT: LBB54_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB54_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB54_1 ; CHECK-NOLSE-O0-NEXT: b LBB54_5 @@ -4719,9 +4719,9 @@ define i32 @atomicrmw_max_i32(ptr %ptr, i32 %rhs) { ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, gt ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas4_rel ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #8] ; 4-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: subs w8, w0, w8 ; CHECK-OUTLINE-O0-NEXT: cset w8, eq +; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: tbz w8, #0, LBB54_1 ; CHECK-OUTLINE-O0-NEXT: b LBB54_2 @@ -4800,9 +4800,9 @@ define i32 @atomicrmw_umin_i32(ptr %ptr, i32 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB55_2 ; CHECK-NOLSE-O0-NEXT: LBB55_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB55_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB55_1 ; CHECK-NOLSE-O0-NEXT: b LBB55_5 @@ -4833,9 +4833,9 @@ define i32 @atomicrmw_umin_i32(ptr %ptr, i32 %rhs) { ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, ls ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas4_acq_rel ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #8] ; 4-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: subs w8, w0, w8 ; CHECK-OUTLINE-O0-NEXT: cset w8, eq +; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: tbz w8, #0, LBB55_1 ; CHECK-OUTLINE-O0-NEXT: b LBB55_2 @@ -4914,9 +4914,9 @@ define i32 @atomicrmw_umax_i32(ptr %ptr, i32 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB56_2 ; CHECK-NOLSE-O0-NEXT: LBB56_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB56_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB56_1 ; CHECK-NOLSE-O0-NEXT: b LBB56_5 @@ -4947,9 +4947,9 @@ define i32 @atomicrmw_umax_i32(ptr %ptr, i32 %rhs) { ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, hi ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas4_relax ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #8] ; 4-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: subs w8, w0, w8 ; CHECK-OUTLINE-O0-NEXT: cset w8, eq +; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: tbz w8, #0, LBB56_1 ; CHECK-OUTLINE-O0-NEXT: b LBB56_2 @@ -5026,9 +5026,9 @@ define i64 @atomicrmw_add_i64(ptr %ptr, i64 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB57_2 ; CHECK-NOLSE-O0-NEXT: LBB57_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB57_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB57_1 ; CHECK-NOLSE-O0-NEXT: b LBB57_5 @@ -5117,9 +5117,9 @@ define i64 @atomicrmw_xchg_i64(ptr %ptr, i64 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB58_2 ; CHECK-NOLSE-O0-NEXT: LBB58_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB58_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB58_1 ; CHECK-NOLSE-O0-NEXT: b LBB58_5 @@ -5210,9 +5210,9 @@ define i64 @atomicrmw_sub_i64(ptr %ptr, i64 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB59_2 ; CHECK-NOLSE-O0-NEXT: LBB59_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB59_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB59_1 ; CHECK-NOLSE-O0-NEXT: b LBB59_5 @@ -5307,9 +5307,9 @@ define i64 @atomicrmw_and_i64(ptr %ptr, i64 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB60_2 ; CHECK-NOLSE-O0-NEXT: LBB60_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB60_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB60_1 ; CHECK-NOLSE-O0-NEXT: b LBB60_5 @@ -5404,9 +5404,9 @@ define i64 @atomicrmw_or_i64(ptr %ptr, i64 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB61_2 ; CHECK-NOLSE-O0-NEXT: LBB61_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB61_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB61_1 ; CHECK-NOLSE-O0-NEXT: b LBB61_5 @@ -5497,9 +5497,9 @@ define i64 @atomicrmw_xor_i64(ptr %ptr, i64 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB62_2 ; CHECK-NOLSE-O0-NEXT: LBB62_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB62_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB62_1 ; CHECK-NOLSE-O0-NEXT: b LBB62_5 @@ -5592,9 +5592,9 @@ define i64 @atomicrmw_min_i64(ptr %ptr, i64 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB63_2 ; CHECK-NOLSE-O0-NEXT: LBB63_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB63_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB63_1 ; CHECK-NOLSE-O0-NEXT: b LBB63_5 @@ -5625,9 +5625,9 @@ define i64 @atomicrmw_min_i64(ptr %ptr, i64 %rhs) { ; CHECK-OUTLINE-O0-NEXT: csel x1, x0, x8, le ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas8_acq ; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #8] ; 8-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: subs x8, x0, x8 ; CHECK-OUTLINE-O0-NEXT: cset w8, eq +; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: tbz w8, #0, LBB63_1 ; CHECK-OUTLINE-O0-NEXT: b LBB63_2 @@ -5706,9 +5706,9 @@ define i64 @atomicrmw_max_i64(ptr %ptr, i64 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB64_2 ; CHECK-NOLSE-O0-NEXT: LBB64_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB64_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB64_1 ; CHECK-NOLSE-O0-NEXT: b LBB64_5 @@ -5739,9 +5739,9 @@ define i64 @atomicrmw_max_i64(ptr %ptr, i64 %rhs) { ; CHECK-OUTLINE-O0-NEXT: csel x1, x0, x8, gt ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas8_rel ; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #8] ; 8-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: subs x8, x0, x8 ; CHECK-OUTLINE-O0-NEXT: cset w8, eq +; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: tbz w8, #0, LBB64_1 ; CHECK-OUTLINE-O0-NEXT: b LBB64_2 @@ -5820,9 +5820,9 @@ define i64 @atomicrmw_umin_i64(ptr %ptr, i64 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB65_2 ; CHECK-NOLSE-O0-NEXT: LBB65_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB65_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB65_1 ; CHECK-NOLSE-O0-NEXT: b LBB65_5 @@ -5853,9 +5853,9 @@ define i64 @atomicrmw_umin_i64(ptr %ptr, i64 %rhs) { ; CHECK-OUTLINE-O0-NEXT: csel x1, x0, x8, ls ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas8_acq_rel ; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #8] ; 8-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: subs x8, x0, x8 ; CHECK-OUTLINE-O0-NEXT: cset w8, eq +; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: tbz w8, #0, LBB65_1 ; CHECK-OUTLINE-O0-NEXT: b LBB65_2 @@ -5934,9 +5934,9 @@ define i64 @atomicrmw_umax_i64(ptr %ptr, i64 %rhs) { ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB66_2 ; CHECK-NOLSE-O0-NEXT: LBB66_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB66_1 Depth=1 -; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: cset w8, eq +; CHECK-NOLSE-O0-NEXT: str x9, [sp] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB66_1 ; CHECK-NOLSE-O0-NEXT: b LBB66_5 @@ -5967,9 +5967,9 @@ define i64 @atomicrmw_umax_i64(ptr %ptr, i64 %rhs) { ; CHECK-OUTLINE-O0-NEXT: csel x1, x0, x8, hi ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas8_relax ; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #8] ; 8-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: subs x8, x0, x8 ; CHECK-OUTLINE-O0-NEXT: cset w8, eq +; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: tbz w8, #0, LBB66_1 ; CHECK-OUTLINE-O0-NEXT: b LBB66_2 diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll index 8aea944b55c2..ceef0c49a45e 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll @@ -65,22 +65,17 @@ define <8 x i16> @combine_vec_udiv_nonuniform(<8 x i16> %x) { ; GISEL-NEXT: ushl v1.8h, v0.8h, v1.8h ; GISEL-NEXT: umull2 v3.4s, v1.8h, v2.8h ; GISEL-NEXT: umull v1.4s, v1.4h, v2.4h -; GISEL-NEXT: uzp2 v1.8h, v1.8h, v3.8h -; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI1_1] +; GISEL-NEXT: ldr q2, [x8, :lo12:.LCPI1_1] ; GISEL-NEXT: adrp x8, .LCPI1_0 -; GISEL-NEXT: sub v2.8h, v0.8h, v1.8h -; GISEL-NEXT: umull2 v4.4s, v2.8h, v3.8h -; GISEL-NEXT: umull v2.4s, v2.4h, v3.4h -; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI1_0] -; GISEL-NEXT: adrp x8, .LCPI1_4 -; GISEL-NEXT: uzp2 v2.8h, v2.8h, v4.8h -; GISEL-NEXT: ldr q4, [x8, :lo12:.LCPI1_4] -; GISEL-NEXT: add v1.8h, v2.8h, v1.8h -; GISEL-NEXT: neg v2.8h, v3.8h -; GISEL-NEXT: movi v3.8h, #1 -; GISEL-NEXT: ushl v1.8h, v1.8h, v2.8h -; GISEL-NEXT: cmeq v2.8h, v4.8h, v3.8h -; GISEL-NEXT: bif v0.16b, v1.16b, v2.16b +; GISEL-NEXT: uzp2 v1.8h, v1.8h, v3.8h +; GISEL-NEXT: sub v0.8h, v0.8h, v1.8h +; GISEL-NEXT: umull2 v3.4s, v0.8h, v2.8h +; GISEL-NEXT: umull v0.4s, v0.4h, v2.4h +; GISEL-NEXT: ldr q2, [x8, :lo12:.LCPI1_0] +; GISEL-NEXT: uzp2 v0.8h, v0.8h, v3.8h +; GISEL-NEXT: add v0.8h, v0.8h, v1.8h +; GISEL-NEXT: neg v1.8h, v2.8h +; GISEL-NEXT: ushl v0.8h, v0.8h, v1.8h ; GISEL-NEXT: ret %1 = udiv <8 x i16> %x, ret <8 x i16> %1 @@ -107,21 +102,16 @@ define <8 x i16> @combine_vec_udiv_nonuniform2(<8 x i16> %x) { ; GISEL-NEXT: adrp x8, .LCPI2_2 ; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI2_2] ; GISEL-NEXT: adrp x8, .LCPI2_1 -; GISEL-NEXT: ldr q2, [x8, :lo12:.LCPI2_1] +; GISEL-NEXT: neg v1.8h, v1.8h +; GISEL-NEXT: ushl v0.8h, v0.8h, v1.8h +; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI2_1] ; GISEL-NEXT: adrp x8, .LCPI2_0 +; GISEL-NEXT: umull2 v2.4s, v0.8h, v1.8h +; GISEL-NEXT: umull v0.4s, v0.4h, v1.4h +; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI2_0] ; GISEL-NEXT: neg v1.8h, v1.8h -; GISEL-NEXT: ushl v1.8h, v0.8h, v1.8h -; GISEL-NEXT: umull2 v3.4s, v1.8h, v2.8h -; GISEL-NEXT: umull v1.4s, v1.4h, v2.4h -; GISEL-NEXT: ldr q2, [x8, :lo12:.LCPI2_0] -; GISEL-NEXT: adrp x8, .LCPI2_3 -; GISEL-NEXT: neg v2.8h, v2.8h -; GISEL-NEXT: ldr q4, [x8, :lo12:.LCPI2_3] -; GISEL-NEXT: uzp2 v1.8h, v1.8h, v3.8h -; GISEL-NEXT: movi v3.8h, #1 -; GISEL-NEXT: ushl v1.8h, v1.8h, v2.8h -; GISEL-NEXT: cmeq v2.8h, v4.8h, v3.8h -; GISEL-NEXT: bif v0.16b, v1.16b, v2.16b +; GISEL-NEXT: uzp2 v0.8h, v0.8h, v2.8h +; GISEL-NEXT: ushl v0.8h, v0.8h, v1.8h ; GISEL-NEXT: ret %1 = udiv <8 x i16> %x, ret <8 x i16> %1 @@ -145,21 +135,16 @@ define <8 x i16> @combine_vec_udiv_nonuniform3(<8 x i16> %x) { ; GISEL-LABEL: combine_vec_udiv_nonuniform3: ; GISEL: // %bb.0: ; GISEL-NEXT: adrp x8, .LCPI3_1 -; GISEL-NEXT: movi v3.8h, #1 ; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI3_1] ; GISEL-NEXT: adrp x8, .LCPI3_0 ; GISEL-NEXT: umull2 v2.4s, v0.8h, v1.8h ; GISEL-NEXT: umull v1.4s, v0.4h, v1.4h ; GISEL-NEXT: uzp2 v1.8h, v1.8h, v2.8h -; GISEL-NEXT: sub v2.8h, v0.8h, v1.8h -; GISEL-NEXT: usra v1.8h, v2.8h, #1 -; GISEL-NEXT: ldr q2, [x8, :lo12:.LCPI3_0] -; GISEL-NEXT: adrp x8, .LCPI3_2 -; GISEL-NEXT: ldr q4, [x8, :lo12:.LCPI3_2] -; GISEL-NEXT: neg v2.8h, v2.8h -; GISEL-NEXT: ushl v1.8h, v1.8h, v2.8h -; GISEL-NEXT: cmeq v2.8h, v4.8h, v3.8h -; GISEL-NEXT: bif v0.16b, v1.16b, v2.16b +; GISEL-NEXT: sub v0.8h, v0.8h, v1.8h +; GISEL-NEXT: usra v1.8h, v0.8h, #1 +; GISEL-NEXT: ldr q0, [x8, :lo12:.LCPI3_0] +; GISEL-NEXT: neg v0.8h, v0.8h +; GISEL-NEXT: ushl v0.8h, v1.8h, v0.8h ; GISEL-NEXT: ret %1 = udiv <8 x i16> %x, ret <8 x i16> %1 @@ -184,19 +169,19 @@ define <16 x i8> @combine_vec_udiv_nonuniform4(<16 x i8> %x) { ; ; GISEL-LABEL: combine_vec_udiv_nonuniform4: ; GISEL: // %bb.0: +; GISEL-NEXT: adrp x8, .LCPI4_2 +; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI4_2] ; GISEL-NEXT: adrp x8, .LCPI4_1 -; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI4_1] +; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI4_1] ; GISEL-NEXT: adrp x8, .LCPI4_0 -; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI4_0] -; GISEL-NEXT: adrp x8, .LCPI4_2 ; GISEL-NEXT: umull2 v2.8h, v0.16b, v1.16b ; GISEL-NEXT: umull v1.8h, v0.8b, v1.8b -; GISEL-NEXT: ldr q4, [x8, :lo12:.LCPI4_2] +; GISEL-NEXT: ldr q4, [x8, :lo12:.LCPI4_0] ; GISEL-NEXT: uzp2 v1.16b, v1.16b, v2.16b ; GISEL-NEXT: neg v2.16b, v3.16b -; GISEL-NEXT: movi v3.16b, #1 +; GISEL-NEXT: shl v3.16b, v4.16b, #7 ; GISEL-NEXT: ushl v1.16b, v1.16b, v2.16b -; GISEL-NEXT: cmeq v2.16b, v4.16b, v3.16b +; GISEL-NEXT: sshr v2.16b, v3.16b, #7 ; GISEL-NEXT: bif v0.16b, v1.16b, v2.16b ; GISEL-NEXT: ret %div = udiv <16 x i8> %x, @@ -232,10 +217,10 @@ define <8 x i16> @pr38477(<8 x i16> %a0) { ; ; GISEL-LABEL: pr38477: ; GISEL: // %bb.0: +; GISEL-NEXT: adrp x8, .LCPI5_3 +; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI5_3] ; GISEL-NEXT: adrp x8, .LCPI5_2 -; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI5_2] -; GISEL-NEXT: adrp x8, .LCPI5_1 -; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI5_1] +; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI5_2] ; GISEL-NEXT: adrp x8, .LCPI5_0 ; GISEL-NEXT: umull2 v2.4s, v0.8h, v1.8h ; GISEL-NEXT: umull v1.4s, v0.4h, v1.4h @@ -243,15 +228,16 @@ define <8 x i16> @pr38477(<8 x i16> %a0) { ; GISEL-NEXT: sub v2.8h, v0.8h, v1.8h ; GISEL-NEXT: umull2 v4.4s, v2.8h, v3.8h ; GISEL-NEXT: umull v2.4s, v2.4h, v3.4h -; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI5_0] -; GISEL-NEXT: adrp x8, .LCPI5_3 +; GISEL-NEXT: ldr d3, [x8, :lo12:.LCPI5_0] +; GISEL-NEXT: adrp x8, .LCPI5_1 +; GISEL-NEXT: ushll v3.8h, v3.8b, #0 ; GISEL-NEXT: uzp2 v2.8h, v2.8h, v4.8h -; GISEL-NEXT: ldr q4, [x8, :lo12:.LCPI5_3] +; GISEL-NEXT: ldr q4, [x8, :lo12:.LCPI5_1] +; GISEL-NEXT: shl v3.8h, v3.8h, #15 ; GISEL-NEXT: add v1.8h, v2.8h, v1.8h -; GISEL-NEXT: neg v2.8h, v3.8h -; GISEL-NEXT: movi v3.8h, #1 +; GISEL-NEXT: neg v2.8h, v4.8h ; GISEL-NEXT: ushl v1.8h, v1.8h, v2.8h -; GISEL-NEXT: cmeq v2.8h, v4.8h, v3.8h +; GISEL-NEXT: sshr v2.8h, v3.8h, #15 ; GISEL-NEXT: bif v0.16b, v1.16b, v2.16b ; GISEL-NEXT: ret %1 = udiv <8 x i16> %a0, diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.mir index ee33b9c50cbe..02233b9f498b 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.mir @@ -6,7 +6,9 @@ body: | bb.1: liveins: $w0 ; CHECK-LABEL: name: udiv_by_scalar_const - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; CHECK: liveins: $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 818089009 ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 @@ -68,44 +70,32 @@ body: | ; CHECK: liveins: $q0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 23 - ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 34 - ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -23 - ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 56 - ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 128 - ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1 - ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 -256 - ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768 - ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16), [[C2]](s16), [[C3]](s16), [[C4]](s16), [[C5]](s16), [[C6]](s16), [[C7]](s16) - ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 0 - ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s16) = G_CONSTANT i16 25645 - ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s16) = G_CONSTANT i16 4 - ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s16) = G_CONSTANT i16 -3855 - ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s16) = G_CONSTANT i16 5 - ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s16) = G_CONSTANT i16 8195 - ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s16) = G_CONSTANT i16 13 - ; CHECK-NEXT: [[C15:%[0-9]+]]:_(s16) = G_CONSTANT i16 3 - ; CHECK-NEXT: [[C16:%[0-9]+]]:_(s16) = G_CONSTANT i16 9363 - ; CHECK-NEXT: [[C17:%[0-9]+]]:_(s16) = G_CONSTANT i16 512 - ; CHECK-NEXT: [[C18:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32767 - ; CHECK-NEXT: [[C19:%[0-9]+]]:_(s16) = G_CONSTANT i16 15 - ; CHECK-NEXT: [[C20:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32639 - ; CHECK-NEXT: [[C21:%[0-9]+]]:_(s16) = G_CONSTANT i16 2 - ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C15]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16) - ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C9]](s16), [[C11]](s16), [[C13]](s16), [[C16]](s16), [[C17]](s16), [[C18]](s16), [[C20]](s16), [[C21]](s16) - ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C7]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16) - ; CHECK-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C10]](s16), [[C12]](s16), [[C14]](s16), [[C8]](s16), [[C8]](s16), [[C19]](s16), [[C19]](s16), [[C8]](s16) - ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[COPY]], [[BUILD_VECTOR1]](<8 x s16>) - ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[LSHR]], [[BUILD_VECTOR2]] + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 0 + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 25645 + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 4 + ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 -3855 + ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 5 + ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 8195 + ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 13 + ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 3 + ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s16) = G_CONSTANT i16 9363 + ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s16) = G_CONSTANT i16 512 + ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32767 + ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s16) = G_CONSTANT i16 15 + ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32639 + ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s16) = G_CONSTANT i16 2 + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16), [[C1]](s16), [[C8]](s16), [[C1]](s16), [[C1]](s16), [[C1]](s16), [[C1]](s16) + ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C2]](s16), [[C4]](s16), [[C6]](s16), [[C9]](s16), [[C10]](s16), [[C11]](s16), [[C13]](s16), [[C14]](s16) + ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16), [[C1]](s16), [[C1]](s16), [[C1]](s16), [[C1]](s16), [[C1]](s16), [[C1]](s16) + ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C3]](s16), [[C5]](s16), [[C7]](s16), [[C1]](s16), [[C1]](s16), [[C12]](s16), [[C12]](s16), [[C1]](s16) + ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[COPY]], [[BUILD_VECTOR]](<8 x s16>) + ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[LSHR]], [[BUILD_VECTOR1]] ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(<8 x s16>) = G_SUB [[COPY]], [[UMULH]] - ; CHECK-NEXT: [[UMULH1:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[SUB]], [[BUILD_VECTOR3]] + ; CHECK-NEXT: [[UMULH1:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[SUB]], [[BUILD_VECTOR2]] ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(<8 x s16>) = G_ADD [[UMULH1]], [[UMULH]] - ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[ADD]], [[BUILD_VECTOR4]](<8 x s16>) - ; CHECK-NEXT: [[C22:%[0-9]+]]:_(s16) = G_CONSTANT i16 1 - ; CHECK-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C22]](s16), [[C22]](s16), [[C22]](s16), [[C22]](s16), [[C22]](s16), [[C22]](s16), [[C22]](s16), [[C22]](s16) - ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<8 x s1>) = G_ICMP intpred(eq), [[BUILD_VECTOR]](<8 x s16>), [[BUILD_VECTOR5]] - ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<8 x s16>) = G_SELECT [[ICMP]](<8 x s1>), [[COPY]], [[LSHR1]] - ; CHECK-NEXT: $q0 = COPY [[SELECT]](<8 x s16>) + ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[ADD]], [[BUILD_VECTOR3]](<8 x s16>) + ; CHECK-NEXT: $q0 = COPY [[LSHR1]](<8 x s16>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 %0:_(<8 x s16>) = COPY $q0 %2:_(s16) = G_CONSTANT i16 23 @@ -136,38 +126,26 @@ body: | ; CHECK: liveins: $q0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 -34 - ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 35 - ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 36 - ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -37 - ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 38 - ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 -39 - ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 40 - ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 -41 - ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16), [[C2]](s16), [[C3]](s16), [[C4]](s16), [[C5]](s16), [[C6]](s16), [[C7]](s16) - ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 1 - ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s16) = G_CONSTANT i16 16393 - ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s16) = G_CONSTANT i16 0 - ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s16) = G_CONSTANT i16 13 - ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s16) = G_CONSTANT i16 -5617 - ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s16) = G_CONSTANT i16 5 - ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s16) = G_CONSTANT i16 -7281 - ; CHECK-NEXT: [[C15:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32749 - ; CHECK-NEXT: [[C16:%[0-9]+]]:_(s16) = G_CONSTANT i16 15 - ; CHECK-NEXT: [[C17:%[0-9]+]]:_(s16) = G_CONSTANT i16 -10347 - ; CHECK-NEXT: [[C18:%[0-9]+]]:_(s16) = G_CONSTANT i16 8197 - ; CHECK-NEXT: [[C19:%[0-9]+]]:_(s16) = G_CONSTANT i16 -13107 - ; CHECK-NEXT: [[C20:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32747 - ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C8]](s16), [[C10]](s16), [[C10]](s16), [[C10]](s16), [[C10]](s16), [[C10]](s16), [[C10]](s16), [[C10]](s16) - ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C9]](s16), [[C12]](s16), [[C14]](s16), [[C15]](s16), [[C17]](s16), [[C18]](s16), [[C19]](s16), [[C20]](s16) - ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C11]](s16), [[C13]](s16), [[C13]](s16), [[C16]](s16), [[C13]](s16), [[C11]](s16), [[C13]](s16), [[C16]](s16) - ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[COPY]], [[BUILD_VECTOR1]](<8 x s16>) - ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[LSHR]], [[BUILD_VECTOR2]] - ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[UMULH]], [[BUILD_VECTOR3]](<8 x s16>) - ; CHECK-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16) - ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<8 x s1>) = G_ICMP intpred(eq), [[BUILD_VECTOR]](<8 x s16>), [[BUILD_VECTOR4]] - ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<8 x s16>) = G_SELECT [[ICMP]](<8 x s1>), [[COPY]], [[LSHR1]] - ; CHECK-NEXT: $q0 = COPY [[SELECT]](<8 x s16>) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 16393 + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 0 + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 13 + ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 -5617 + ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 5 + ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 -7281 + ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32749 + ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 15 + ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s16) = G_CONSTANT i16 -10347 + ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s16) = G_CONSTANT i16 8197 + ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s16) = G_CONSTANT i16 -13107 + ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32747 + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C2]](s16), [[C2]](s16), [[C2]](s16), [[C2]](s16), [[C2]](s16), [[C2]](s16), [[C2]](s16) + ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C4]](s16), [[C6]](s16), [[C7]](s16), [[C9]](s16), [[C10]](s16), [[C11]](s16), [[C12]](s16) + ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C3]](s16), [[C5]](s16), [[C5]](s16), [[C8]](s16), [[C5]](s16), [[C3]](s16), [[C5]](s16), [[C8]](s16) + ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[COPY]], [[BUILD_VECTOR]](<8 x s16>) + ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[LSHR]], [[BUILD_VECTOR1]] + ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[UMULH]], [[BUILD_VECTOR2]](<8 x s16>) + ; CHECK-NEXT: $q0 = COPY [[LSHR1]](<8 x s16>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 %0:_(<8 x s16>) = COPY $q0 %2:_(s16) = G_CONSTANT i16 -34 @@ -198,39 +176,28 @@ body: | ; CHECK: liveins: $q0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 7 - ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 23 - ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 25 - ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 27 - ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 31 - ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 47 - ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 63 - ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 127 - ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16), [[C2]](s16), [[C3]](s16), [[C4]](s16), [[C5]](s16), [[C6]](s16), [[C7]](s16) - ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 9363 - ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s16) = G_CONSTANT i16 2 - ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s16) = G_CONSTANT i16 25645 - ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s16) = G_CONSTANT i16 4 - ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s16) = G_CONSTANT i16 18351 - ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s16) = G_CONSTANT i16 12137 - ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s16) = G_CONSTANT i16 2115 - ; CHECK-NEXT: [[C15:%[0-9]+]]:_(s16) = G_CONSTANT i16 23705 - ; CHECK-NEXT: [[C16:%[0-9]+]]:_(s16) = G_CONSTANT i16 5 - ; CHECK-NEXT: [[C17:%[0-9]+]]:_(s16) = G_CONSTANT i16 1041 - ; CHECK-NEXT: [[C18:%[0-9]+]]:_(s16) = G_CONSTANT i16 517 - ; CHECK-NEXT: [[C19:%[0-9]+]]:_(s16) = G_CONSTANT i16 6 - ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C8]](s16), [[C10]](s16), [[C12]](s16), [[C13]](s16), [[C14]](s16), [[C15]](s16), [[C17]](s16), [[C18]](s16) - ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C9]](s16), [[C11]](s16), [[C11]](s16), [[C11]](s16), [[C11]](s16), [[C16]](s16), [[C16]](s16), [[C19]](s16) - ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[COPY]], [[BUILD_VECTOR1]] + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9363 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 2 + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 25645 + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 4 + ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 18351 + ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 12137 + ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 2115 + ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 23705 + ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 5 + ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s16) = G_CONSTANT i16 1041 + ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s16) = G_CONSTANT i16 517 + ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s16) = G_CONSTANT i16 6 + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C2]](s16), [[C4]](s16), [[C5]](s16), [[C6]](s16), [[C7]](s16), [[C9]](s16), [[C10]](s16) + ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C3]](s16), [[C3]](s16), [[C3]](s16), [[C3]](s16), [[C8]](s16), [[C8]](s16), [[C11]](s16) + ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[COPY]], [[BUILD_VECTOR]] ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(<8 x s16>) = G_SUB [[COPY]], [[UMULH]] - ; CHECK-NEXT: [[C20:%[0-9]+]]:_(s16) = G_CONSTANT i16 1 - ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C20]](s16), [[C20]](s16), [[C20]](s16), [[C20]](s16), [[C20]](s16), [[C20]](s16), [[C20]](s16), [[C20]](s16) - ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[SUB]], [[BUILD_VECTOR3]](<8 x s16>) + ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s16) = G_CONSTANT i16 1 + ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C12]](s16), [[C12]](s16), [[C12]](s16), [[C12]](s16), [[C12]](s16), [[C12]](s16), [[C12]](s16), [[C12]](s16) + ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[SUB]], [[BUILD_VECTOR2]](<8 x s16>) ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(<8 x s16>) = G_ADD [[LSHR]], [[UMULH]] - ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[ADD]], [[BUILD_VECTOR2]](<8 x s16>) - ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<8 x s1>) = G_ICMP intpred(eq), [[BUILD_VECTOR]](<8 x s16>), [[BUILD_VECTOR3]] - ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<8 x s16>) = G_SELECT [[ICMP]](<8 x s1>), [[COPY]], [[LSHR1]] - ; CHECK-NEXT: $q0 = COPY [[SELECT]](<8 x s16>) + ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[ADD]], [[BUILD_VECTOR1]](<8 x s16>) + ; CHECK-NEXT: $q0 = COPY [[LSHR1]](<8 x s16>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 %0:_(<8 x s16>) = COPY $q0 %2:_(s16) = G_CONSTANT i16 7 @@ -261,19 +228,17 @@ body: | ; CHECK: liveins: $q0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<16 x s8>) = COPY $q0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 -64 - ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s8) = G_CONSTANT i8 1 - ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[C]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8) - ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s8) = G_CONSTANT i8 0 - ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s8) = G_CONSTANT i8 -85 - ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s8) = G_CONSTANT i8 7 - ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[C3]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8) - ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[C4]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8) - ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<16 x s8>) = G_UMULH [[COPY]], [[BUILD_VECTOR1]] - ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<16 x s8>) = G_LSHR [[UMULH]], [[BUILD_VECTOR2]](<16 x s8>) - ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8), [[C1]](s8) - ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<16 x s1>) = G_ICMP intpred(eq), [[BUILD_VECTOR]](<16 x s8>), [[BUILD_VECTOR3]] - ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<16 x s8>) = G_SELECT [[ICMP]](<16 x s1>), [[COPY]], [[LSHR]] + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 0 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s8) = G_CONSTANT i8 -85 + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s8) = G_CONSTANT i8 7 + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[C1]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8) + ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[C2]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8) + ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<16 x s8>) = G_UMULH [[COPY]], [[BUILD_VECTOR]] + ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<16 x s8>) = G_LSHR [[UMULH]], [[BUILD_VECTOR1]](<16 x s8>) + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 false + ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true + ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<16 x s1>) = G_BUILD_VECTOR [[C3]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1), [[C4]](s1) + ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<16 x s8>) = G_SELECT [[BUILD_VECTOR2]](<16 x s1>), [[COPY]], [[LSHR]] ; CHECK-NEXT: $q0 = COPY [[SELECT]](<16 x s8>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 %0:_(<16 x s8>) = COPY $q0 @@ -299,39 +264,31 @@ body: | ; CHECK: liveins: $q0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1 - ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 119 - ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 73 - ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -111 - ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 -3 - ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 118 - ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 32 - ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 31 - ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16), [[C2]](s16), [[C3]](s16), [[C4]](s16), [[C5]](s16), [[C6]](s16), [[C7]](s16) - ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 0 - ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s16) = G_CONSTANT i16 4957 - ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768 - ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s16) = G_CONSTANT i16 6 - ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s16) = G_CONSTANT i16 -8079 - ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s16) = G_CONSTANT i16 4103 - ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s16) = G_CONSTANT i16 12 - ; CHECK-NEXT: [[C15:%[0-9]+]]:_(s16) = G_CONSTANT i16 16385 - ; CHECK-NEXT: [[C16:%[0-9]+]]:_(s16) = G_CONSTANT i16 14 - ; CHECK-NEXT: [[C17:%[0-9]+]]:_(s16) = G_CONSTANT i16 -29991 - ; CHECK-NEXT: [[C18:%[0-9]+]]:_(s16) = G_CONSTANT i16 2048 - ; CHECK-NEXT: [[C19:%[0-9]+]]:_(s16) = G_CONSTANT i16 2115 - ; CHECK-NEXT: [[C20:%[0-9]+]]:_(s16) = G_CONSTANT i16 4 - ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C8]](s16), [[C9]](s16), [[C12]](s16), [[C13]](s16), [[C15]](s16), [[C17]](s16), [[C18]](s16), [[C19]](s16) - ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C8]](s16), [[C10]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C8]](s16), [[C10]](s16) - ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C8]](s16), [[C11]](s16), [[C11]](s16), [[C14]](s16), [[C16]](s16), [[C11]](s16), [[C8]](s16), [[C20]](s16) - ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[COPY]], [[BUILD_VECTOR1]] + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 0 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 4957 + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768 + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 6 + ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 -8079 + ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 4103 + ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 12 + ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 16385 + ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 14 + ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s16) = G_CONSTANT i16 -29991 + ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s16) = G_CONSTANT i16 2048 + ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s16) = G_CONSTANT i16 2115 + ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s16) = G_CONSTANT i16 4 + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16), [[C4]](s16), [[C5]](s16), [[C7]](s16), [[C9]](s16), [[C10]](s16), [[C11]](s16) + ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C2]](s16), [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16), [[C2]](s16) + ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C3]](s16), [[C3]](s16), [[C6]](s16), [[C8]](s16), [[C3]](s16), [[C]](s16), [[C12]](s16) + ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[COPY]], [[BUILD_VECTOR]] ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(<8 x s16>) = G_SUB [[COPY]], [[UMULH]] - ; CHECK-NEXT: [[UMULH1:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[SUB]], [[BUILD_VECTOR2]] + ; CHECK-NEXT: [[UMULH1:%[0-9]+]]:_(<8 x s16>) = G_UMULH [[SUB]], [[BUILD_VECTOR1]] ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(<8 x s16>) = G_ADD [[UMULH1]], [[UMULH]] - ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[ADD]], [[BUILD_VECTOR3]](<8 x s16>) - ; CHECK-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16) - ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<8 x s1>) = G_ICMP intpred(eq), [[BUILD_VECTOR]](<8 x s16>), [[BUILD_VECTOR4]] - ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<8 x s16>) = G_SELECT [[ICMP]](<8 x s1>), [[COPY]], [[LSHR]] + ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<8 x s16>) = G_LSHR [[ADD]], [[BUILD_VECTOR2]](<8 x s16>) + ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s1) = G_CONSTANT i1 true + ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s1) = G_CONSTANT i1 false + ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<8 x s1>) = G_BUILD_VECTOR [[C13]](s1), [[C14]](s1), [[C14]](s1), [[C14]](s1), [[C14]](s1), [[C14]](s1), [[C14]](s1), [[C14]](s1) + ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<8 x s16>) = G_SELECT [[BUILD_VECTOR3]](<8 x s1>), [[COPY]], [[LSHR]] ; CHECK-NEXT: $q0 = COPY [[SELECT]](<8 x s16>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 %0:_(<8 x s16>) = COPY $q0 diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cmpxchg-128.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cmpxchg-128.mir index 6a6e0b63b103..26230efbbe86 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cmpxchg-128.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cmpxchg-128.mir @@ -12,22 +12,6 @@ body: | liveins: $x0, $x1, $x2, $x3, $x4 - ; CHECK-LABEL: name: compare_swap_128 - ; CHECK: liveins: $x0_x1, $x1, $x0, $x1, $x2, $x3, $x4 - ; CHECK: [[COPY:%[0-9]+]]:gpr64(p0) = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1 - ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2 - ; CHECK: [[COPY3:%[0-9]+]]:_(s64) = COPY $x3 - ; CHECK: [[COPY4:%[0-9]+]]:_(s64) = COPY $x4 - ; CHECK: [[COPY5:%[0-9]+]]:gpr64(s64) = COPY [[COPY1]](s64) - ; CHECK: [[COPY6:%[0-9]+]]:gpr64(s64) = COPY [[COPY2]](s64) - ; CHECK: [[COPY7:%[0-9]+]]:gpr64(s64) = COPY [[COPY3]](s64) - ; CHECK: [[COPY8:%[0-9]+]]:gpr64(s64) = COPY [[COPY4]](s64) - ; CHECK: early-clobber %13:gpr64(s64), early-clobber %14:gpr64(s64), early-clobber %16:gpr32common = CMP_SWAP_128_ACQUIRE [[COPY]](p0), [[COPY5]](s64), [[COPY6]](s64), [[COPY7]](s64), [[COPY8]](s64) :: (load store acquire acquire 16) - ; CHECK: [[COPY9:%[0-9]+]]:gpr64 = COPY %16 - ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES %13(s64), %14(s64) - ; CHECK: G_STORE [[MV]](s128), [[COPY]](p0) :: (store 16) - ; CHECK: RET_ReallyLR ; CHECK-NOLSE-LABEL: name: compare_swap_128 ; CHECK-NOLSE: liveins: $x0_x1, $x1, $x0, $x1, $x2, $x3, $x4 ; CHECK-NOLSE-NEXT: {{ $}} @@ -40,11 +24,13 @@ body: | ; CHECK-NOLSE-NEXT: [[COPY6:%[0-9]+]]:gpr64(s64) = COPY [[COPY2]](s64) ; CHECK-NOLSE-NEXT: [[COPY7:%[0-9]+]]:gpr64(s64) = COPY [[COPY3]](s64) ; CHECK-NOLSE-NEXT: [[COPY8:%[0-9]+]]:gpr64(s64) = COPY [[COPY4]](s64) - ; CHECK-NOLSE-NEXT: early-clobber %13:gpr64common(s64), early-clobber %14:gpr64common(s64), early-clobber %16:gpr32common = CMP_SWAP_128_ACQUIRE [[COPY]](p0), [[COPY5]](s64), [[COPY6]](s64), [[COPY7]](s64), [[COPY8]](s64) :: (load store acquire acquire (s128)) - ; CHECK-NOLSE-NEXT: [[COPY9:%[0-9]+]]:gpr64 = COPY %16 - ; CHECK-NOLSE-NEXT: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES %13(s64), %14(s64) - ; CHECK-NOLSE-NEXT: G_STORE [[MV]](s128), [[COPY]](p0) :: (store (s128)) + ; CHECK-NOLSE-NEXT: early-clobber %14:gpr64common(s64), early-clobber %15:gpr64common(s64), early-clobber %17:gpr32common = CMP_SWAP_128_ACQUIRE [[COPY]](p0), [[COPY5]](s64), [[COPY6]](s64), [[COPY7]](s64), [[COPY8]](s64) :: (load store acquire acquire (s128)) + ; CHECK-NOLSE-NEXT: [[COPY9:%[0-9]+]]:gpr64 = COPY %17 + ; CHECK-NOLSE-NEXT: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES %14(s64), %15(s64) + ; CHECK-NOLSE-NEXT: [[COPY10:%[0-9]+]]:_(s128) = COPY [[MV]](s128) + ; CHECK-NOLSE-NEXT: G_STORE [[COPY10]](s128), [[COPY]](p0) :: (store (s128)) ; CHECK-NOLSE-NEXT: RET_ReallyLR + ; ; CHECK-LSE-LABEL: name: compare_swap_128 ; CHECK-LSE: liveins: $x0_x1, $x1, $x0, $x1, $x2, $x3, $x4 ; CHECK-LSE-NEXT: {{ $}} @@ -59,7 +45,8 @@ body: | ; CHECK-LSE-NEXT: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[CASPAX]](s128), 0 ; CHECK-LSE-NEXT: [[EXTRACT1:%[0-9]+]]:_(s64) = G_EXTRACT [[CASPAX]](s128), 64 ; CHECK-LSE-NEXT: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[EXTRACT]](s64), [[EXTRACT1]](s64) - ; CHECK-LSE-NEXT: G_STORE [[MV]](s128), [[COPY]](p0) :: (store (s128)) + ; CHECK-LSE-NEXT: [[COPY5:%[0-9]+]]:_(s128) = COPY [[MV]](s128) + ; CHECK-LSE-NEXT: G_STORE [[COPY5]](s128), [[COPY]](p0) :: (store (s128)) ; CHECK-LSE-NEXT: RET_ReallyLR %0:_(p0) = COPY $x0 %3:_(s64) = COPY $x1 diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cmpxchg-with-success.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cmpxchg-with-success.mir index 3c010789a2b7..05e6212af062 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cmpxchg-with-success.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cmpxchg-with-success.mir @@ -16,13 +16,16 @@ body: | liveins: $x0 ; CHECK-LABEL: name: cmpxchg_i32 - ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 - ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 - ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 - ; CHECK: [[ATOMIC_CMPXCHG:%[0-9]+]]:_(s32) = G_ATOMIC_CMPXCHG [[COPY]](p0), [[C]], [[C1]] :: (load store monotonic (s64) on %ir.addr) - ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ATOMIC_CMPXCHG]](s32), [[C]] - ; CHECK: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[ATOMIC_CMPXCHG]], [[ICMP]] - ; CHECK: $w0 = COPY [[MUL]](s32) + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK-NEXT: [[ATOMIC_CMPXCHG:%[0-9]+]]:_(s32) = G_ATOMIC_CMPXCHG [[COPY]](p0), [[C]], [[C1]] :: (load store monotonic (s64) on %ir.addr) + ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ATOMIC_CMPXCHG]](s32), [[C]] + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[ATOMIC_CMPXCHG]](s32) + ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY1]], [[ICMP]] + ; CHECK-NEXT: $w0 = COPY [[MUL]](s32) %0:_(p0) = COPY $x0 %1:_(s32) = G_CONSTANT i32 0 %2:_(s32) = G_CONSTANT i32 1 @@ -40,14 +43,17 @@ body: | liveins: $x0 ; CHECK-LABEL: name: cmpxchg_i64 - ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 - ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 - ; CHECK: [[ATOMIC_CMPXCHG:%[0-9]+]]:_(s64) = G_ATOMIC_CMPXCHG [[COPY]](p0), [[C]], [[C1]] :: (load store monotonic (s64) on %ir.addr) - ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ATOMIC_CMPXCHG]](s64), [[C]] - ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ICMP]](s32) - ; CHECK: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ATOMIC_CMPXCHG]], [[ANYEXT]] - ; CHECK: $x0 = COPY [[MUL]](s64) + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; CHECK-NEXT: [[ATOMIC_CMPXCHG:%[0-9]+]]:_(s64) = G_ATOMIC_CMPXCHG [[COPY]](p0), [[C]], [[C1]] :: (load store monotonic (s64) on %ir.addr) + ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ATOMIC_CMPXCHG]](s64), [[C]] + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY [[ATOMIC_CMPXCHG]](s64) + ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ICMP]](s32) + ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[COPY1]], [[ANYEXT]] + ; CHECK-NEXT: $x0 = COPY [[MUL]](s64) %0:_(p0) = COPY $x0 %1:_(s64) = G_CONSTANT i64 0 %2:_(s64) = G_CONSTANT i64 1 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomic-cmpxchg-with-success.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomic-cmpxchg-with-success.mir index e288d9d5ab3c..eafd1e15e2cb 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomic-cmpxchg-with-success.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomic-cmpxchg-with-success.mir @@ -16,7 +16,8 @@ body: | ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY1]](s32) ; CHECK-NEXT: [[AMDGPU_ATOMIC_CMPXCHG:%[0-9]+]]:_(s32) = G_AMDGPU_ATOMIC_CMPXCHG [[COPY]](p1), [[BUILD_VECTOR]] :: (load store syncscope("agent-one-as") monotonic monotonic (s32), addrspace 1) ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[AMDGPU_ATOMIC_CMPXCHG]](s32), [[COPY1]] - ; CHECK-NEXT: S_ENDPGM 0, implicit [[AMDGPU_ATOMIC_CMPXCHG]](s32), implicit [[ICMP]](s1) + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AMDGPU_ATOMIC_CMPXCHG]](s32) + ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY3]](s32), implicit [[ICMP]](s1) %0:_(p1) = COPY $vgpr0_vgpr1 %1:_(s32) = COPY $vgpr2 %2:_(s32) = COPY $vgpr3 @@ -40,7 +41,8 @@ body: | ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY1]](s32) ; CHECK-NEXT: [[AMDGPU_ATOMIC_CMPXCHG:%[0-9]+]]:_(s32) = G_AMDGPU_ATOMIC_CMPXCHG [[COPY]](p0), [[BUILD_VECTOR]] :: (load store syncscope("agent-one-as") monotonic monotonic (s32)) ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[AMDGPU_ATOMIC_CMPXCHG]](s32), [[COPY1]] - ; CHECK-NEXT: S_ENDPGM 0, implicit [[AMDGPU_ATOMIC_CMPXCHG]](s32), implicit [[ICMP]](s1) + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AMDGPU_ATOMIC_CMPXCHG]](s32) + ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY3]](s32), implicit [[ICMP]](s1) %0:_(p0) = COPY $vgpr0_vgpr1 %1:_(s32) = COPY $vgpr2 %2:_(s32) = COPY $vgpr3 @@ -63,7 +65,8 @@ body: | ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2 ; CHECK-NEXT: [[ATOMIC_CMPXCHG:%[0-9]+]]:_(s32) = G_ATOMIC_CMPXCHG [[COPY]](p3), [[COPY1]], [[COPY2]] :: (load store syncscope("agent-one-as") monotonic monotonic (s32), addrspace 3) ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[ATOMIC_CMPXCHG]](s32), [[COPY1]] - ; CHECK-NEXT: S_ENDPGM 0, implicit [[ATOMIC_CMPXCHG]](s32), implicit [[ICMP]](s1) + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[ATOMIC_CMPXCHG]](s32) + ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY3]](s32), implicit [[ICMP]](s1) %0:_(p3) = COPY $vgpr0 %1:_(s32) = COPY $vgpr1 %2:_(s32) = COPY $vgpr2 @@ -87,7 +90,8 @@ body: | ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[COPY2]](s64), [[COPY1]](s64) ; CHECK-NEXT: [[AMDGPU_ATOMIC_CMPXCHG:%[0-9]+]]:_(s64) = G_AMDGPU_ATOMIC_CMPXCHG [[COPY]](p1), [[BUILD_VECTOR]] :: (load store syncscope("agent-one-as") monotonic monotonic (s64), addrspace 1) ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[AMDGPU_ATOMIC_CMPXCHG]](s64), [[COPY1]] - ; CHECK-NEXT: S_ENDPGM 0, implicit [[AMDGPU_ATOMIC_CMPXCHG]](s64), implicit [[ICMP]](s1) + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY [[AMDGPU_ATOMIC_CMPXCHG]](s64) + ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY3]](s64), implicit [[ICMP]](s1) %0:_(p1) = COPY $vgpr0_vgpr1 %1:_(s64) = COPY $vgpr2_vgpr3 %2:_(s64) = COPY $vgpr4_vgpr5 @@ -110,7 +114,8 @@ body: | ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr3_vgpr4 ; CHECK-NEXT: [[ATOMIC_CMPXCHG:%[0-9]+]]:_(s64) = G_ATOMIC_CMPXCHG [[COPY]](p3), [[COPY1]], [[COPY2]] :: (load store syncscope("agent-one-as") monotonic monotonic (s64), addrspace 3) ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[ATOMIC_CMPXCHG]](s64), [[COPY1]] - ; CHECK-NEXT: S_ENDPGM 0, implicit [[ATOMIC_CMPXCHG]](s64), implicit [[ICMP]](s1) + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY [[ATOMIC_CMPXCHG]](s64) + ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY3]](s64), implicit [[ICMP]](s1) %0:_(p3) = COPY $vgpr0 %1:_(s64) = COPY $vgpr1_vgpr2 %2:_(s64) = COPY $vgpr3_vgpr4 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir index dba20e128237..eb86a981c9f1 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir @@ -86,8 +86,9 @@ body: | ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[ADD]](s32), [[COPY]] ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[COPY1]](s32), [[C]] ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[XOR]](s1) - ; CHECK-NEXT: $vgpr0 = COPY [[ADD]](s32) + ; CHECK-NEXT: $vgpr0 = COPY [[COPY2]](s32) ; CHECK-NEXT: $vgpr1 = COPY [[ZEXT]](s32) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr1 @@ -117,8 +118,9 @@ body: | ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]] ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[COPY1]](s64), [[C]] ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[XOR]](s1) - ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64) + ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY2]](s64) ; CHECK-NEXT: $vgpr2 = COPY [[ZEXT]](s32) %0:_(s64) = COPY $vgpr0_vgpr1 %1:_(s64) = COPY $vgpr2_vgpr3 @@ -172,11 +174,12 @@ body: | ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP1]] ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR]](s1) ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1) + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY [[BITCAST2]](<2 x s16>) ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]] ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]] ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND2]](s32), [[AND3]](s32) - ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>) + ; CHECK-NEXT: $vgpr0 = COPY [[COPY3]](<2 x s16>) ; CHECK-NEXT: $vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<2 x s32>) %0:_(<2 x s16>) = COPY $vgpr0 %1:_(<2 x s16>) = COPY $vgpr1 @@ -360,13 +363,14 @@ body: | ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1) ; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR2]](s1) ; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR3]](s1) + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(<4 x s16>) = COPY [[CONCAT_VECTORS]](<4 x s16>) ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]] ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]] ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]] ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]] ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[AND5]](s32), [[AND6]](s32), [[AND7]](s32) - ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>) + ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY5]](<4 x s16>) ; CHECK-NEXT: $vgpr2_vgpr3_vgpr4_vgpr5 = COPY [[BUILD_VECTOR]](<4 x s32>) %0:_(<4 x s16>) = COPY $vgpr0_vgpr1 %1:_(<4 x s16>) = COPY $vgpr1_vgpr2 @@ -403,11 +407,12 @@ body: | ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP1]] ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR]](s1) ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY [[BUILD_VECTOR]](<2 x s32>) ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C1]] ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C1]] ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[AND1]](s32) - ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY2]](<2 x s32>) ; CHECK-NEXT: $vgpr2_vgpr3 = COPY [[BUILD_VECTOR1]](<2 x s32>) %0:_(<2 x s32>) = COPY $vgpr0_vgpr1 %1:_(<2 x s32>) = COPY $vgpr2_vgpr3 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir index 93d00714158b..80b3166108ad 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir @@ -955,15 +955,16 @@ body: | ; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]] ; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[COPY1]](s64), [[C]] ; GFX6-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX6-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX6-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX6-NEXT: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX6-NEXT: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]] ; GFX6-NEXT: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]] ; GFX6-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32) - ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[SELECT]](s64) ; ; GFX8-LABEL: name: saddsat_s64 @@ -980,15 +981,16 @@ body: | ; GFX8-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]] ; GFX8-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[COPY1]](s64), [[C]] ; GFX8-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX8-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX8-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX8-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX8-NEXT: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX8-NEXT: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]] ; GFX8-NEXT: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]] ; GFX8-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32) - ; GFX8-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX8-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[SELECT]](s64) ; ; GFX9-LABEL: name: saddsat_s64 @@ -1005,15 +1007,16 @@ body: | ; GFX9-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]] ; GFX9-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[COPY1]](s64), [[C]] ; GFX9-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX9-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX9-NEXT: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX9-NEXT: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]] ; GFX9-NEXT: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]] ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32) - ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[SELECT]](s64) %0:_(s64) = COPY $vgpr0_vgpr1 %1:_(s64) = COPY $vgpr2_vgpr3 @@ -1043,15 +1046,16 @@ body: | ; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]] ; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV2]](s64), [[C]] ; GFX6-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX6-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX6-NEXT: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX6-NEXT: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX6-NEXT: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]] ; GFX6-NEXT: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]] ; GFX6-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32) - ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX6-NEXT: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64) ; GFX6-NEXT: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64) ; GFX6-NEXT: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[UV12]], [[UV14]] @@ -1060,13 +1064,14 @@ body: | ; GFX6-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]] ; GFX6-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV3]](s64), [[C]] ; GFX6-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]] - ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32) + ; GFX6-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY [[MV2]](s64) + ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY3]], [[C1]](s32) ; GFX6-NEXT: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64) ; GFX6-NEXT: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX6-NEXT: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]] ; GFX6-NEXT: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO7]] ; GFX6-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO6]](s32), [[UADDE6]](s32) - ; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]] + ; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[COPY3]] ; GFX6-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64) ; GFX6-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>) ; @@ -1086,15 +1091,16 @@ body: | ; GFX8-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]] ; GFX8-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV2]](s64), [[C]] ; GFX8-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX8-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX8-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX8-NEXT: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX8-NEXT: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX8-NEXT: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]] ; GFX8-NEXT: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]] ; GFX8-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32) - ; GFX8-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX8-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX8-NEXT: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64) ; GFX8-NEXT: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64) ; GFX8-NEXT: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[UV12]], [[UV14]] @@ -1103,13 +1109,14 @@ body: | ; GFX8-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]] ; GFX8-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV3]](s64), [[C]] ; GFX8-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]] - ; GFX8-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32) + ; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY [[MV2]](s64) + ; GFX8-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY3]], [[C1]](s32) ; GFX8-NEXT: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64) ; GFX8-NEXT: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX8-NEXT: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]] ; GFX8-NEXT: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO7]] ; GFX8-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO6]](s32), [[UADDE6]](s32) - ; GFX8-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]] + ; GFX8-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[COPY3]] ; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64) ; GFX8-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>) ; @@ -1129,15 +1136,16 @@ body: | ; GFX9-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]] ; GFX9-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV2]](s64), [[C]] ; GFX9-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX9-NEXT: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX9-NEXT: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX9-NEXT: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]] ; GFX9-NEXT: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]] ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32) - ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX9-NEXT: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64) ; GFX9-NEXT: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64) ; GFX9-NEXT: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[UV12]], [[UV14]] @@ -1146,13 +1154,14 @@ body: | ; GFX9-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]] ; GFX9-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV3]](s64), [[C]] ; GFX9-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]] - ; GFX9-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32) + ; GFX9-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY [[MV2]](s64) + ; GFX9-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY3]], [[C1]](s32) ; GFX9-NEXT: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64) ; GFX9-NEXT: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX9-NEXT: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]] ; GFX9-NEXT: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO7]] ; GFX9-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO6]](s32), [[UADDE6]](s32) - ; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]] + ; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[COPY3]] ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64) ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>) %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir index 57b1ab9b194e..220450c5e4ec 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir @@ -86,8 +86,9 @@ body: | ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SUB]](s32), [[COPY]] ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY1]](s32), [[C]] ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[SUB]](s32) ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[XOR]](s1) - ; CHECK-NEXT: $vgpr0 = COPY [[SUB]](s32) + ; CHECK-NEXT: $vgpr0 = COPY [[COPY2]](s32) ; CHECK-NEXT: $vgpr1 = COPY [[ZEXT]](s32) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr1 @@ -117,8 +118,9 @@ body: | ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]] ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY1]](s64), [[C]] ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[XOR]](s1) - ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64) + ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY2]](s64) ; CHECK-NEXT: $vgpr2 = COPY [[ZEXT]](s32) %0:_(s64) = COPY $vgpr0_vgpr1 %1:_(s64) = COPY $vgpr2_vgpr3 @@ -172,11 +174,12 @@ body: | ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP1]] ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR]](s1) ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1) + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY [[BITCAST2]](<2 x s16>) ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]] ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]] ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND2]](s32), [[AND3]](s32) - ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>) + ; CHECK-NEXT: $vgpr0 = COPY [[COPY3]](<2 x s16>) ; CHECK-NEXT: $vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<2 x s32>) %0:_(<2 x s16>) = COPY $vgpr0 %1:_(<2 x s16>) = COPY $vgpr1 @@ -360,13 +363,14 @@ body: | ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1) ; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR2]](s1) ; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR3]](s1) + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(<4 x s16>) = COPY [[CONCAT_VECTORS]](<4 x s16>) ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]] ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]] ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]] ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]] ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[AND5]](s32), [[AND6]](s32), [[AND7]](s32) - ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>) + ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY5]](<4 x s16>) ; CHECK-NEXT: $vgpr2_vgpr3_vgpr4_vgpr5 = COPY [[BUILD_VECTOR]](<4 x s32>) %0:_(<4 x s16>) = COPY $vgpr0_vgpr1 %1:_(<4 x s16>) = COPY $vgpr1_vgpr2 @@ -403,11 +407,12 @@ body: | ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP1]] ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR]](s1) ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY [[BUILD_VECTOR]](<2 x s32>) ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C1]] ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C1]] ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[AND1]](s32) - ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY2]](<2 x s32>) ; CHECK-NEXT: $vgpr2_vgpr3 = COPY [[BUILD_VECTOR1]](<2 x s32>) %0:_(<2 x s32>) = COPY $vgpr0_vgpr1 %1:_(<2 x s32>) = COPY $vgpr2_vgpr3 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir index 33a8cda8e84b..49fb6e9bdaf3 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir @@ -955,15 +955,16 @@ body: | ; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]] ; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY1]](s64), [[C]] ; GFX6-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX6-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX6-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX6-NEXT: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX6-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]] ; GFX6-NEXT: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]] ; GFX6-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32) - ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[SELECT]](s64) ; ; GFX8-LABEL: name: ssubsat_s64 @@ -980,15 +981,16 @@ body: | ; GFX8-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]] ; GFX8-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY1]](s64), [[C]] ; GFX8-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX8-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX8-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX8-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX8-NEXT: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX8-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]] ; GFX8-NEXT: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]] ; GFX8-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32) - ; GFX8-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX8-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[SELECT]](s64) ; ; GFX9-LABEL: name: ssubsat_s64 @@ -1005,15 +1007,16 @@ body: | ; GFX9-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]] ; GFX9-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY1]](s64), [[C]] ; GFX9-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX9-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX9-NEXT: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX9-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]] ; GFX9-NEXT: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]] ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32) - ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[SELECT]](s64) %0:_(s64) = COPY $vgpr0_vgpr1 %1:_(s64) = COPY $vgpr2_vgpr3 @@ -1043,15 +1046,16 @@ body: | ; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]] ; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV2]](s64), [[C]] ; GFX6-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX6-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX6-NEXT: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX6-NEXT: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX6-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]] ; GFX6-NEXT: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO1]] ; GFX6-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32) - ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX6-NEXT: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64) ; GFX6-NEXT: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64) ; GFX6-NEXT: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV12]], [[UV14]] @@ -1060,13 +1064,14 @@ body: | ; GFX6-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]] ; GFX6-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV3]](s64), [[C]] ; GFX6-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]] - ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32) + ; GFX6-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY [[MV2]](s64) + ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY3]], [[C1]](s32) ; GFX6-NEXT: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64) ; GFX6-NEXT: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX6-NEXT: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]] ; GFX6-NEXT: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO3]] ; GFX6-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32) - ; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]] + ; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[COPY3]] ; GFX6-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64) ; GFX6-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>) ; @@ -1086,15 +1091,16 @@ body: | ; GFX8-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]] ; GFX8-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV2]](s64), [[C]] ; GFX8-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX8-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX8-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX8-NEXT: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX8-NEXT: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX8-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]] ; GFX8-NEXT: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO1]] ; GFX8-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32) - ; GFX8-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX8-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX8-NEXT: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64) ; GFX8-NEXT: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64) ; GFX8-NEXT: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV12]], [[UV14]] @@ -1103,13 +1109,14 @@ body: | ; GFX8-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]] ; GFX8-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV3]](s64), [[C]] ; GFX8-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]] - ; GFX8-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32) + ; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY [[MV2]](s64) + ; GFX8-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY3]], [[C1]](s32) ; GFX8-NEXT: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64) ; GFX8-NEXT: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX8-NEXT: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]] ; GFX8-NEXT: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO3]] ; GFX8-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32) - ; GFX8-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]] + ; GFX8-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[COPY3]] ; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64) ; GFX8-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>) ; @@ -1129,15 +1136,16 @@ body: | ; GFX9-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]] ; GFX9-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV2]](s64), [[C]] ; GFX9-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]] + ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[MV]](s64) ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63 - ; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32) + ; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C1]](s32) ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808 ; GFX9-NEXT: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64) ; GFX9-NEXT: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX9-NEXT: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]] ; GFX9-NEXT: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO1]] ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32) - ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]] + ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[COPY2]] ; GFX9-NEXT: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64) ; GFX9-NEXT: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64) ; GFX9-NEXT: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV12]], [[UV14]] @@ -1146,13 +1154,14 @@ body: | ; GFX9-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]] ; GFX9-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV3]](s64), [[C]] ; GFX9-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]] - ; GFX9-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32) + ; GFX9-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY [[MV2]](s64) + ; GFX9-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY3]], [[C1]](s32) ; GFX9-NEXT: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64) ; GFX9-NEXT: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64) ; GFX9-NEXT: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]] ; GFX9-NEXT: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO3]] ; GFX9-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32) - ; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]] + ; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[COPY3]] ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64) ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>) %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3 diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/add.mir b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/add.mir index 52352edbe339..e471e1047caa 100644 --- a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/add.mir +++ b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/add.mir @@ -220,10 +220,12 @@ body: | ; MIPS32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $a3 ; MIPS32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY3]], [[COPY1]] ; MIPS32-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[COPY1]] + ; MIPS32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) ; MIPS32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY2]], [[COPY]] ; MIPS32-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ICMP]] - ; MIPS32-NEXT: $v0 = COPY [[ADD2]](s32) - ; MIPS32-NEXT: $v1 = COPY [[ADD]](s32) + ; MIPS32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32) + ; MIPS32-NEXT: $v0 = COPY [[COPY5]](s32) + ; MIPS32-NEXT: $v1 = COPY [[COPY4]](s32) ; MIPS32-NEXT: RetRA implicit $v0, implicit $v1 %2:_(s32) = COPY $a0 %3:_(s32) = COPY $a1 @@ -268,6 +270,7 @@ body: | ; MIPS32-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p0) :: (load (s32) from %fixed-stack.3) ; MIPS32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD]], [[COPY]] ; MIPS32-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[COPY]] + ; MIPS32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) ; MIPS32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[COPY1]] ; MIPS32-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD1]](s32), [[LOAD1]] ; MIPS32-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ICMP]] @@ -275,6 +278,7 @@ body: | ; MIPS32-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ADD2]](s32), [[C]] ; MIPS32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ICMP2]], [[ICMP]] ; MIPS32-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ICMP1]], [[AND]] + ; MIPS32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32) ; MIPS32-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[LOAD2]], [[COPY2]] ; MIPS32-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD3]](s32), [[LOAD2]] ; MIPS32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 @@ -283,13 +287,15 @@ body: | ; MIPS32-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ADD4]](s32), [[C]] ; MIPS32-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ICMP4]], [[OR]] ; MIPS32-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ICMP3]], [[AND2]] + ; MIPS32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY [[ADD4]](s32) ; MIPS32-NEXT: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[LOAD3]], [[COPY3]] ; MIPS32-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C1]] ; MIPS32-NEXT: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[ADD5]], [[AND3]] - ; MIPS32-NEXT: $v0 = COPY [[ADD]](s32) - ; MIPS32-NEXT: $v1 = COPY [[ADD2]](s32) - ; MIPS32-NEXT: $a0 = COPY [[ADD4]](s32) - ; MIPS32-NEXT: $a1 = COPY [[ADD6]](s32) + ; MIPS32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY [[ADD6]](s32) + ; MIPS32-NEXT: $v0 = COPY [[COPY4]](s32) + ; MIPS32-NEXT: $v1 = COPY [[COPY5]](s32) + ; MIPS32-NEXT: $a0 = COPY [[COPY6]](s32) + ; MIPS32-NEXT: $a1 = COPY [[COPY7]](s32) ; MIPS32-NEXT: RetRA implicit $v0, implicit $v1, implicit $a0, implicit $a1 %2:_(s32) = COPY $a0 %3:_(s32) = COPY $a1 @@ -331,10 +337,11 @@ body: | ; MIPS32-NEXT: [[COPY3:%[0-9]+]]:_(p0) = COPY $a3 ; MIPS32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]] ; MIPS32-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[COPY1]] + ; MIPS32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) ; MIPS32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; MIPS32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C]] ; MIPS32-NEXT: G_STORE [[AND]](s32), [[COPY3]](p0) :: (store (s8) into %ir.pcarry_flag) - ; MIPS32-NEXT: G_STORE [[ADD]](s32), [[COPY2]](p0) :: (store (s32) into %ir.padd) + ; MIPS32-NEXT: G_STORE [[COPY4]](s32), [[COPY2]](p0) :: (store (s32) into %ir.padd) ; MIPS32-NEXT: RetRA %0:_(s32) = COPY $a0 %1:_(s32) = COPY $a1 diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/cttz.mir b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/cttz.mir index 3e7bcdc39d5d..a06bb6da45d2 100644 --- a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/cttz.mir +++ b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/cttz.mir @@ -139,9 +139,11 @@ body: | ; MIPS32-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[ADD1]], [[SUB1]] ; MIPS32-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C]] ; MIPS32-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD3]](s32), [[C]] + ; MIPS32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ADD3]](s32) ; MIPS32-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[C1]], [[C1]] ; MIPS32-NEXT: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[ADD4]], [[ICMP1]] - ; MIPS32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[ADD3]](s32), [[ADD5]](s32) + ; MIPS32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[ADD5]](s32) + ; MIPS32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32) ; MIPS32-NEXT: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[C1]] ; MIPS32-NEXT: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[COPY1]], [[C1]] ; MIPS32-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[XOR2]], [[XOR3]] diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mul.mir b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mul.mir index 7ad286b952cb..674d7b68bfae 100644 --- a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mul.mir +++ b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mul.mir @@ -275,8 +275,10 @@ body: | ; MIPS32-NEXT: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[LOAD]], [[COPY]] ; MIPS32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]] ; MIPS32-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[MUL2]] - ; MIPS32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]] + ; MIPS32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) + ; MIPS32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY4]], [[UMULH]] ; MIPS32-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD1]](s32), [[UMULH]] + ; MIPS32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD1]](s32) ; MIPS32-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ICMP]], [[ICMP1]] ; MIPS32-NEXT: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[LOAD2]], [[COPY]] ; MIPS32-NEXT: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[LOAD1]], [[COPY1]] @@ -285,17 +287,22 @@ body: | ; MIPS32-NEXT: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[LOAD]], [[COPY1]] ; MIPS32-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[MUL3]], [[MUL4]] ; MIPS32-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD3]](s32), [[MUL4]] - ; MIPS32-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[MUL5]] + ; MIPS32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY [[ADD3]](s32) + ; MIPS32-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[COPY6]], [[MUL5]] ; MIPS32-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD4]](s32), [[MUL5]] + ; MIPS32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY [[ADD4]](s32) ; MIPS32-NEXT: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[ICMP2]], [[ICMP3]] - ; MIPS32-NEXT: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[ADD4]], [[UMULH1]] + ; MIPS32-NEXT: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[COPY7]], [[UMULH1]] ; MIPS32-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD6]](s32), [[UMULH1]] + ; MIPS32-NEXT: [[COPY8:%[0-9]+]]:_(s32) = COPY [[ADD6]](s32) ; MIPS32-NEXT: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[ADD5]], [[ICMP4]] - ; MIPS32-NEXT: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD6]], [[UMULH2]] + ; MIPS32-NEXT: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[COPY8]], [[UMULH2]] ; MIPS32-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD8]](s32), [[UMULH2]] + ; MIPS32-NEXT: [[COPY9:%[0-9]+]]:_(s32) = COPY [[ADD8]](s32) ; MIPS32-NEXT: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[ICMP5]] - ; MIPS32-NEXT: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ADD8]], [[ADD2]] + ; MIPS32-NEXT: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[COPY9]], [[ADD2]] ; MIPS32-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD10]](s32), [[ADD2]] + ; MIPS32-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY [[ADD10]](s32) ; MIPS32-NEXT: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD9]], [[ICMP6]] ; MIPS32-NEXT: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[LOAD3]], [[COPY]] ; MIPS32-NEXT: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[LOAD2]], [[COPY1]] @@ -312,8 +319,8 @@ body: | ; MIPS32-NEXT: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[ADD16]], [[UMULH5]] ; MIPS32-NEXT: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[ADD11]] ; MIPS32-NEXT: $v0 = COPY [[MUL]](s32) - ; MIPS32-NEXT: $v1 = COPY [[ADD1]](s32) - ; MIPS32-NEXT: $a0 = COPY [[ADD10]](s32) + ; MIPS32-NEXT: $v1 = COPY [[COPY5]](s32) + ; MIPS32-NEXT: $a0 = COPY [[COPY10]](s32) ; MIPS32-NEXT: $a1 = COPY [[ADD18]](s32) ; MIPS32-NEXT: RetRA implicit $v0, implicit $v1, implicit $a0, implicit $a1 %2:_(s32) = COPY $a0 @@ -359,23 +366,28 @@ body: | ; MIPS32-NEXT: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[COPY2]], [[COPY]] ; MIPS32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL]], [[MUL1]] ; MIPS32-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[MUL1]] - ; MIPS32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]] + ; MIPS32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) + ; MIPS32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY4]], [[UMULH]] ; MIPS32-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD1]](s32), [[UMULH]] + ; MIPS32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD1]](s32) ; MIPS32-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ICMP]], [[ICMP1]] ; MIPS32-NEXT: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[COPY3]], [[COPY1]] ; MIPS32-NEXT: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[COPY3]], [[COPY]] ; MIPS32-NEXT: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[COPY2]], [[COPY1]] ; MIPS32-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[MUL2]], [[UMULH1]] ; MIPS32-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD3]](s32), [[UMULH1]] - ; MIPS32-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[UMULH2]] + ; MIPS32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY [[ADD3]](s32) + ; MIPS32-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[COPY6]], [[UMULH2]] ; MIPS32-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD4]](s32), [[UMULH2]] + ; MIPS32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY [[ADD4]](s32) ; MIPS32-NEXT: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[ICMP2]], [[ICMP3]] - ; MIPS32-NEXT: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[ADD4]], [[ADD2]] + ; MIPS32-NEXT: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[COPY7]], [[ADD2]] ; MIPS32-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD6]](s32), [[ADD2]] + ; MIPS32-NEXT: [[COPY8:%[0-9]+]]:_(s32) = COPY [[ADD6]](s32) ; MIPS32-NEXT: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[ADD5]], [[ICMP4]] ; MIPS32-NEXT: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[COPY3]], [[COPY1]] ; MIPS32-NEXT: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[ADD7]] - ; MIPS32-NEXT: $v0 = COPY [[ADD6]](s32) + ; MIPS32-NEXT: $v0 = COPY [[COPY8]](s32) ; MIPS32-NEXT: $v1 = COPY [[ADD8]](s32) ; MIPS32-NEXT: RetRA implicit $v0, implicit $v1 %2:_(s32) = COPY $a0 diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-add-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-add-rv32.mir index d169eb316dfc..b3c62df4ffdc 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-add-rv32.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-add-rv32.mir @@ -89,10 +89,12 @@ body: | ; CHECK-NEXT: %yhi:_(s32) = COPY $x13 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD %xlo, %ylo ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), %ylo + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD %xhi, %yhi ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ICMP]] - ; CHECK-NEXT: $x10 = COPY [[ADD]](s32) - ; CHECK-NEXT: $x11 = COPY [[ADD2]](s32) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32) + ; CHECK-NEXT: $x10 = COPY [[COPY]](s32) + ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11 %xlo:_(s32) = COPY $x10 %xhi:_(s32) = COPY $x11 @@ -121,10 +123,12 @@ body: | ; CHECK-NEXT: %hi2:_(s32) = COPY $x13 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD %lo1, %lo2 ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), %lo2 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD %hi1, %hi2 ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ICMP]] - ; CHECK-NEXT: $x10 = COPY [[ADD]](s32) - ; CHECK-NEXT: $x11 = COPY [[ADD2]](s32) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32) + ; CHECK-NEXT: $x10 = COPY [[COPY]](s32) + ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11 %lo1:_(s32) = COPY $x10 %hi1:_(s32) = COPY $x11 @@ -152,6 +156,7 @@ body: | ; CHECK-NEXT: %hi2:_(s32) = COPY $x15 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD %lo1, %lo2 ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), %lo2 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD %mid1, %mid2 ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD1]](s32), %mid1 ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ICMP]] @@ -159,11 +164,13 @@ body: | ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ADD2]](s32), [[C]] ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ICMP2]], [[ICMP]] ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ICMP1]], [[AND]] + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32) ; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD %hi1, %hi2 ; CHECK-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[OR]] - ; CHECK-NEXT: $x10 = COPY [[ADD]](s32) - ; CHECK-NEXT: $x11 = COPY [[ADD2]](s32) - ; CHECK-NEXT: $x12 = COPY [[ADD4]](s32) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ADD4]](s32) + ; CHECK-NEXT: $x10 = COPY [[COPY]](s32) + ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32) + ; CHECK-NEXT: $x12 = COPY [[COPY2]](s32) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11, implicit $x12 %lo1:_(s32) = COPY $x10 %mid1:_(s32) = COPY $x11 diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-add-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-add-rv64.mir index f394e4d5064e..6e76bb0e3eff 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-add-rv64.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-add-rv64.mir @@ -121,10 +121,12 @@ body: | ; CHECK-NEXT: %y01:_(s64) = COPY $x13 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD %x00, %y00 ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[ADD]](s64), %y00 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY [[ADD]](s64) ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD %x01, %y01 ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s64) = G_ADD [[ADD1]], [[ICMP]] - ; CHECK-NEXT: $x10 = COPY [[ADD]](s64) - ; CHECK-NEXT: $x11 = COPY [[ADD2]](s64) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY [[ADD2]](s64) + ; CHECK-NEXT: $x10 = COPY [[COPY]](s64) + ; CHECK-NEXT: $x11 = COPY [[COPY1]](s64) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11 %x00:_(s64) = COPY $x10 %x01:_(s64) = COPY $x11 @@ -153,10 +155,12 @@ body: | ; CHECK-NEXT: %hi2:_(s64) = COPY $x13 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD %lo1, %lo2 ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[ADD]](s64), %lo2 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY [[ADD]](s64) ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD %hi1, %hi2 ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s64) = G_ADD [[ADD1]], [[ICMP]] - ; CHECK-NEXT: $x10 = COPY [[ADD]](s64) - ; CHECK-NEXT: $x11 = COPY [[ADD2]](s64) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY [[ADD2]](s64) + ; CHECK-NEXT: $x10 = COPY [[COPY]](s64) + ; CHECK-NEXT: $x11 = COPY [[COPY1]](s64) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11 %lo1:_(s64) = COPY $x10 %hi1:_(s64) = COPY $x11 @@ -184,6 +188,7 @@ body: | ; CHECK-NEXT: %hi2:_(s64) = COPY $x15 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD %lo1, %lo2 ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[ADD]](s64), %lo2 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY [[ADD]](s64) ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD %mid1, %mid2 ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[ADD1]](s64), %mid1 ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s64) = G_ADD [[ADD1]], [[ICMP]] @@ -194,14 +199,16 @@ body: | ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[TRUNC1]] ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[ICMP1]](s64) ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[TRUNC2]], [[AND]] + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY [[ADD2]](s64) ; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s64) = G_ADD %hi1, %hi2 ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32) ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C1]] ; CHECK-NEXT: [[ADD4:%[0-9]+]]:_(s64) = G_ADD [[ADD3]], [[AND1]] - ; CHECK-NEXT: $x10 = COPY [[ADD]](s64) - ; CHECK-NEXT: $x11 = COPY [[ADD2]](s64) - ; CHECK-NEXT: $x12 = COPY [[ADD4]](s64) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[ADD4]](s64) + ; CHECK-NEXT: $x10 = COPY [[COPY]](s64) + ; CHECK-NEXT: $x11 = COPY [[COPY1]](s64) + ; CHECK-NEXT: $x12 = COPY [[COPY2]](s64) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11, implicit $x12 %lo1:_(s64) = COPY $x10 %mid1:_(s64) = COPY $x11 diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv32.mir index c348ec6f73ad..9227e6530221 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv32.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv32.mir @@ -92,7 +92,8 @@ body: | ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[ADD]](s32), [[COPY]] ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY1]](s32), [[C]] ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[ICMP1]], [[ICMP]] - ; CHECK-NEXT: $x10 = COPY [[ADD]](s32) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) + ; CHECK-NEXT: $x10 = COPY [[COPY2]](s32) ; CHECK-NEXT: $x11 = COPY [[XOR]](s32) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11 %0:_(s32) = COPY $x10 @@ -119,21 +120,23 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY2]] ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[COPY2]] + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY1]], [[COPY3]] ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ICMP]] + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32) ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 - ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[ADD2]](s32), [[COPY1]] - ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ADD2]](s32), [[COPY1]] - ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[COPY]] + ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY5]](s32), [[COPY1]] + ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY5]](s32), [[COPY1]] + ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY4]](s32), [[COPY]] ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[ICMP3]], [[ICMP1]] ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY3]](s32), [[C1]] ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY3]](s32), [[C1]] ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY2]](s32), [[C]] ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s32), [[ICMP6]], [[ICMP4]] ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[SELECT]] - ; CHECK-NEXT: $x10 = COPY [[ADD]](s32) - ; CHECK-NEXT: $x11 = COPY [[ADD2]](s32) + ; CHECK-NEXT: $x10 = COPY [[COPY4]](s32) + ; CHECK-NEXT: $x11 = COPY [[COPY5]](s32) ; CHECK-NEXT: $x12 = COPY [[XOR]](s32) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11, implicit $x12 %2:_(s32) = COPY $x10 @@ -241,7 +244,8 @@ body: | ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[SUB]](s32), [[COPY]] ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[COPY1]](s32), [[C]] ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[ICMP1]], [[ICMP]] - ; CHECK-NEXT: $x10 = COPY [[SUB]](s32) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[SUB]](s32) + ; CHECK-NEXT: $x10 = COPY [[COPY2]](s32) ; CHECK-NEXT: $x11 = COPY [[XOR]](s32) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11 %0:_(s32) = COPY $x10 @@ -377,7 +381,8 @@ body: | ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]] ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[COPY1]] - ; CHECK-NEXT: $x10 = COPY [[ADD]](s32) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) + ; CHECK-NEXT: $x10 = COPY [[COPY2]](s32) ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11 %0:_(s32) = COPY $x10 @@ -404,14 +409,16 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY2]] ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[COPY2]] + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY1]], [[COPY3]] ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ICMP]] - ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD2]](s32), [[COPY3]] - ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ADD2]](s32), [[COPY3]] - ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[COPY2]] + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32) + ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY5]](s32), [[COPY3]] + ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY5]](s32), [[COPY3]] + ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY4]](s32), [[COPY2]] ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[ICMP3]], [[ICMP1]] - ; CHECK-NEXT: $x10 = COPY [[ADD]](s32) - ; CHECK-NEXT: $x11 = COPY [[ADD2]](s32) + ; CHECK-NEXT: $x10 = COPY [[COPY4]](s32) + ; CHECK-NEXT: $x11 = COPY [[COPY5]](s32) ; CHECK-NEXT: $x12 = COPY [[SELECT]](s32) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11, implicit $x12 %2:_(s32) = COPY $x10 diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv64.mir index 5506f5228e9d..8acaff5dbb25 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv64.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv64.mir @@ -125,8 +125,9 @@ body: | ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ICMP1]](s64) ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[ICMP]](s64) ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[TRUNC1]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[ADD]](s64) ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[XOR]](s32) - ; CHECK-NEXT: $x10 = COPY [[ADD]](s64) + ; CHECK-NEXT: $x10 = COPY [[COPY2]](s64) ; CHECK-NEXT: $x11 = COPY [[ANYEXT]](s64) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11 %0:_(s64) = COPY $x10 @@ -261,8 +262,9 @@ body: | ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ICMP1]](s64) ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[ICMP]](s64) ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[TRUNC1]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[SUB]](s64) ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[XOR]](s32) - ; CHECK-NEXT: $x10 = COPY [[SUB]](s64) + ; CHECK-NEXT: $x10 = COPY [[COPY2]](s64) ; CHECK-NEXT: $x11 = COPY [[ANYEXT]](s64) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11 %0:_(s64) = COPY $x10 @@ -364,7 +366,8 @@ body: | ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295 ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]] ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[ZEXT]](s64), [[AND]] - ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD]](s32) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) + ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY2]](s32) ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64) ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11 @@ -393,7 +396,8 @@ body: | ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[COPY1]] ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[ADD]](s64), [[COPY1]] - ; CHECK-NEXT: $x10 = COPY [[ADD]](s64) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[ADD]](s64) + ; CHECK-NEXT: $x10 = COPY [[COPY2]](s64) ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11 %0:_(s64) = COPY $x10 diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv32.mir index 433d6e6b821f..ec2dc568a5ec 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv32.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv32.mir @@ -162,8 +162,10 @@ body: | ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH %lo1, %lo2 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]] ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[MUL2]] - ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]] + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) + ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[UMULH]] ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD1]](s32), [[UMULH]] + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[ADD1]](s32) ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ICMP]], [[ICMP1]] ; CHECK-NEXT: [[MUL3:%[0-9]+]]:_(s32) = G_MUL %hi1, %lo2 ; CHECK-NEXT: [[MUL4:%[0-9]+]]:_(s32) = G_MUL %mid1, %mid2 @@ -171,13 +173,18 @@ body: | ; CHECK-NEXT: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH %mid1, %lo2 ; CHECK-NEXT: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH %lo1, %mid2 ; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[MUL3]], [[MUL4]] - ; CHECK-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[MUL5]] - ; CHECK-NEXT: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[ADD4]], [[UMULH1]] - ; CHECK-NEXT: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[ADD5]], [[UMULH2]] - ; CHECK-NEXT: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[ADD6]], [[ADD2]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ADD3]](s32) + ; CHECK-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[COPY2]], [[MUL5]] + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[ADD4]](s32) + ; CHECK-NEXT: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[COPY3]], [[UMULH1]] + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD5]](s32) + ; CHECK-NEXT: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[COPY4]], [[UMULH2]] + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD6]](s32) + ; CHECK-NEXT: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[COPY5]], [[ADD2]] + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY [[ADD7]](s32) ; CHECK-NEXT: $x10 = COPY [[MUL]](s32) - ; CHECK-NEXT: $x11 = COPY [[ADD1]](s32) - ; CHECK-NEXT: $x12 = COPY [[ADD7]](s32) + ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32) + ; CHECK-NEXT: $x12 = COPY [[COPY6]](s32) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11, implicit $x12 %lo1:_(s32) = COPY $x10 %mid1:_(s32) = COPY $x11 diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv64.mir index 09e002e8428d..39d9c5b7dfd1 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv64.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv64.mir @@ -194,8 +194,10 @@ body: | ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(s64) = G_UMULH %lo1, %lo2 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[MUL1]], [[MUL2]] ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[ADD]](s64), [[MUL2]] - ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[ADD]], [[UMULH]] + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY [[ADD]](s64) + ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[UMULH]] ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[ADD1]](s64), [[UMULH]] + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY [[ADD1]](s64) ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s64) = G_ADD [[ICMP]], [[ICMP1]] ; CHECK-NEXT: [[MUL3:%[0-9]+]]:_(s64) = G_MUL %hi1, %lo2 ; CHECK-NEXT: [[MUL4:%[0-9]+]]:_(s64) = G_MUL %mid1, %mid2 @@ -203,13 +205,18 @@ body: | ; CHECK-NEXT: [[UMULH1:%[0-9]+]]:_(s64) = G_UMULH %mid1, %lo2 ; CHECK-NEXT: [[UMULH2:%[0-9]+]]:_(s64) = G_UMULH %lo1, %mid2 ; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s64) = G_ADD [[MUL3]], [[MUL4]] - ; CHECK-NEXT: [[ADD4:%[0-9]+]]:_(s64) = G_ADD [[ADD3]], [[MUL5]] - ; CHECK-NEXT: [[ADD5:%[0-9]+]]:_(s64) = G_ADD [[ADD4]], [[UMULH1]] - ; CHECK-NEXT: [[ADD6:%[0-9]+]]:_(s64) = G_ADD [[ADD5]], [[UMULH2]] - ; CHECK-NEXT: [[ADD7:%[0-9]+]]:_(s64) = G_ADD [[ADD6]], [[ADD2]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[ADD3]](s64) + ; CHECK-NEXT: [[ADD4:%[0-9]+]]:_(s64) = G_ADD [[COPY2]], [[MUL5]] + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY [[ADD4]](s64) + ; CHECK-NEXT: [[ADD5:%[0-9]+]]:_(s64) = G_ADD [[COPY3]], [[UMULH1]] + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY [[ADD5]](s64) + ; CHECK-NEXT: [[ADD6:%[0-9]+]]:_(s64) = G_ADD [[COPY4]], [[UMULH2]] + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY [[ADD6]](s64) + ; CHECK-NEXT: [[ADD7:%[0-9]+]]:_(s64) = G_ADD [[COPY5]], [[ADD2]] + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY [[ADD7]](s64) ; CHECK-NEXT: $x10 = COPY [[MUL]](s64) - ; CHECK-NEXT: $x11 = COPY [[ADD1]](s64) - ; CHECK-NEXT: $x12 = COPY [[ADD7]](s64) + ; CHECK-NEXT: $x11 = COPY [[COPY1]](s64) + ; CHECK-NEXT: $x12 = COPY [[COPY6]](s64) ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11, implicit $x12 %lo1:_(s64) = COPY $x10 %mid1:_(s64) = COPY $x11 diff --git a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp index 08857de3cf4e..822707a1f4ed 100644 --- a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp @@ -275,4 +275,174 @@ TEST_F(AArch64GISelMITest, TestConstantFoldCTT) { EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; } +TEST_F(AArch64GISelMITest, TestConstantFoldICMP) { + setUp(); + if (!TM) + GTEST_SKIP(); + + LLT s32 = LLT::scalar(32); + LLT s1 = LLT::scalar(1); + + GISelCSEInfo CSEInfo; + CSEInfo.setCSEConfig(std::make_unique()); + CSEInfo.analyze(*MF); + B.setCSEInfo(&CSEInfo); + CSEMIRBuilder CSEB(B.getState()); + + auto One = CSEB.buildConstant(s32, 1); + auto Two = CSEB.buildConstant(s32, 2); + auto MinusOne = CSEB.buildConstant(s32, -1); + auto MinusTwo = CSEB.buildConstant(s32, -2); + + // ICMP_EQ + { + auto I = CSEB.buildICmp(CmpInst::Predicate::ICMP_EQ, s1, One, One); + EXPECT_TRUE(I->getOpcode() == TargetOpcode::G_CONSTANT); + EXPECT_TRUE(I->getOperand(1).getCImm()->getZExtValue()); + } + + // ICMP_NE + { + auto I = CSEB.buildICmp(CmpInst::Predicate::ICMP_NE, s1, One, Two); + EXPECT_TRUE(I->getOpcode() == TargetOpcode::G_CONSTANT); + EXPECT_TRUE(I->getOperand(1).getCImm()->getZExtValue()); + } + + // ICMP_UGT + { + auto I = CSEB.buildICmp(CmpInst::Predicate::ICMP_UGT, s1, Two, One); + EXPECT_TRUE(I->getOpcode() == TargetOpcode::G_CONSTANT); + EXPECT_TRUE(I->getOperand(1).getCImm()->getZExtValue()); + } + + // ICMP_UGE + { + auto I = CSEB.buildICmp(CmpInst::Predicate::ICMP_UGE, s1, One, One); + EXPECT_TRUE(I->getOpcode() == TargetOpcode::G_CONSTANT); + EXPECT_TRUE(I->getOperand(1).getCImm()->getZExtValue()); + } + + // ICMP_ULT + { + auto I = CSEB.buildICmp(CmpInst::Predicate::ICMP_ULT, s1, One, Two); + EXPECT_TRUE(I->getOpcode() == TargetOpcode::G_CONSTANT); + EXPECT_TRUE(I->getOperand(1).getCImm()->getZExtValue()); + } + + // ICMP_ULE + { + auto I = CSEB.buildICmp(CmpInst::Predicate::ICMP_ULE, s1, Two, Two); + EXPECT_TRUE(I->getOpcode() == TargetOpcode::G_CONSTANT); + EXPECT_TRUE(I->getOperand(1).getCImm()->getZExtValue()); + } + + // ICMP_SGT + { + auto I = + CSEB.buildICmp(CmpInst::Predicate::ICMP_SGT, s1, MinusOne, MinusTwo); + EXPECT_TRUE(I->getOpcode() == TargetOpcode::G_CONSTANT); + EXPECT_TRUE(I->getOperand(1).getCImm()->getZExtValue()); + } + + // ICMP_SGE + { + auto I = + CSEB.buildICmp(CmpInst::Predicate::ICMP_SGE, s1, MinusOne, MinusOne); + EXPECT_TRUE(I->getOpcode() == TargetOpcode::G_CONSTANT); + EXPECT_TRUE(I->getOperand(1).getCImm()->getZExtValue()); + } + + // ICMP_SLT + { + auto I = + CSEB.buildICmp(CmpInst::Predicate::ICMP_SLT, s1, MinusTwo, MinusOne); + EXPECT_TRUE(I->getOpcode() == TargetOpcode::G_CONSTANT); + EXPECT_TRUE(I->getOperand(1).getCImm()->getZExtValue()); + } + + // ICMP_SLE + { + auto I = + CSEB.buildICmp(CmpInst::Predicate::ICMP_SLE, s1, MinusTwo, MinusOne); + EXPECT_TRUE(I->getOpcode() == TargetOpcode::G_CONSTANT); + EXPECT_TRUE(I->getOperand(1).getCImm()->getZExtValue()); + } + + LLT VecTy = LLT::fixed_vector(2, s32); + LLT DstTy = LLT::fixed_vector(2, s1); + auto Three = CSEB.buildConstant(s32, 3); + auto MinusThree = CSEB.buildConstant(s32, -3); + auto OneOne = CSEB.buildBuildVector(VecTy, {One.getReg(0), One.getReg(0)}); + auto OneTwo = CSEB.buildBuildVector(VecTy, {One.getReg(0), Two.getReg(0)}); + auto TwoThree = + CSEB.buildBuildVector(VecTy, {Two.getReg(0), Three.getReg(0)}); + auto MinusOneOne = + CSEB.buildBuildVector(VecTy, {MinusOne.getReg(0), MinusOne.getReg(0)}); + auto MinusOneTwo = + CSEB.buildBuildVector(VecTy, {MinusOne.getReg(0), MinusTwo.getReg(0)}); + auto MinusTwoThree = + CSEB.buildBuildVector(VecTy, {MinusTwo.getReg(0), MinusThree.getReg(0)}); + + // ICMP_EQ + CSEB.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, OneOne, OneOne); + + // ICMP_NE + CSEB.buildICmp(CmpInst::Predicate::ICMP_NE, DstTy, OneOne, OneTwo); + + // ICMP_UGT + CSEB.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, TwoThree, OneTwo); + + // ICMP_UGE + CSEB.buildICmp(CmpInst::Predicate::ICMP_UGE, DstTy, OneTwo, OneOne); + + // ICMP_ULT + CSEB.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, OneOne, OneTwo); + + // ICMP_ULE + CSEB.buildICmp(CmpInst::Predicate::ICMP_ULE, DstTy, OneTwo, OneOne); + + // ICMP_SGT + CSEB.buildICmp(CmpInst::Predicate::ICMP_SGT, DstTy, MinusOneTwo, + MinusTwoThree); + + // ICMP_SGE + CSEB.buildICmp(CmpInst::Predicate::ICMP_SGE, DstTy, MinusOneTwo, MinusOneOne); + + // ICMP_SLT + CSEB.buildICmp(CmpInst::Predicate::ICMP_SLT, DstTy, MinusTwoThree, + MinusOneTwo); + + // ICMP_SLE + CSEB.buildICmp(CmpInst::Predicate::ICMP_SLE, DstTy, MinusOneTwo, MinusOneOne); + + auto CheckStr = R"( + ; CHECK: [[One:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[Two:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 + ; CHECK: [[MinusOne:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1 + ; CHECK: [[MinusTwo:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2 + ; CHECK: [[True:%[0-9]+]]:_(s1) = G_CONSTANT i1 true + ; CHECK: [[Three:%[0-9]+]]:_(s32) = G_CONSTANT i32 3 + ; CHECK: [[MinusThree:%[0-9]+]]:_(s32) = G_CONSTANT i32 -3 + ; CHECK: {{%[0-9]+}}:_(<2 x s32>) = G_BUILD_VECTOR [[One]]:_(s32), [[One]]:_(s32) + ; CHECK: {{%[0-9]+}}:_(<2 x s32>) = G_BUILD_VECTOR [[One]]:_(s32), [[Two]]:_(s32) + ; CHECK: {{%[0-9]+}}:_(<2 x s32>) = G_BUILD_VECTOR [[Two]]:_(s32), [[Three]]:_(s32) + ; CHECK: {{%[0-9]+}}:_(<2 x s32>) = G_BUILD_VECTOR [[MinusOne]]:_(s32), [[MinusOne]]:_(s32) + ; CHECK: {{%[0-9]+}}:_(<2 x s32>) = G_BUILD_VECTOR [[MinusOne]]:_(s32), [[MinusTwo]]:_(s32) + ; CHECK: {{%[0-9]+}}:_(<2 x s32>) = G_BUILD_VECTOR [[MinusTwo]]:_(s32), [[MinusThree]]:_(s32) + ; CHECK: {{%[0-9]+}}:_(<2 x s1>) = G_BUILD_VECTOR [[True]]:_(s1), [[True]]:_(s1) + ; CHECK: [[False:%[0-9]+]]:_(s1) = G_CONSTANT i1 false + ; CHECK: {{%[0-9]+}}:_(<2 x s1>) = G_BUILD_VECTOR [[False]]:_(s1), [[True]]:_(s1) + ; CHECK: {{%[0-9]+}}:_(<2 x s1>) = G_BUILD_VECTOR [[True]]:_(s1), [[True]]:_(s1) + ; CHECK: {{%[0-9]+}}:_(<2 x s1>) = G_BUILD_VECTOR [[True]]:_(s1), [[True]]:_(s1) + ; CHECK: {{%[0-9]+}}:_(<2 x s1>) = G_BUILD_VECTOR [[False]]:_(s1), [[True]]:_(s1) + ; CHECK: {{%[0-9]+}}:_(<2 x s1>) = G_BUILD_VECTOR [[True]]:_(s1), [[False]]:_(s1) + ; CHECK: {{%[0-9]+}}:_(<2 x s1>) = G_BUILD_VECTOR [[True]]:_(s1), [[True]]:_(s1) + ; CHECK: {{%[0-9]+}}:_(<2 x s1>) = G_BUILD_VECTOR [[True]]:_(s1), [[False]]:_(s1) + ; CHECK: {{%[0-9]+}}:_(<2 x s1>) = G_BUILD_VECTOR [[True]]:_(s1), [[True]]:_(s1) + ; CHECK: {{%[0-9]+}}:_(<2 x s1>) = G_BUILD_VECTOR [[True]]:_(s1), [[True]]:_(s1) + )"; + + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + } // namespace -- GitLab From cc8c6b037cd24a47ff6953508e5734562ef1ecc4 Mon Sep 17 00:00:00 2001 From: dhruvachak Date: Fri, 29 Mar 2024 13:33:43 -0700 Subject: [PATCH 0026/1690] [OpenMP] [amdgpu] Added a synchronous version of data exchange. (#87032) Similar to H2D and D2H, use synchronous mode for large data transfers beyond a certain size for D2D as well. As with H2D and D2H, this size is controlled by an env-var. --- .../plugins-nextgen/amdgpu/src/rtl.cpp | 21 ++++++ .../test/offloading/d2d_memcpy_sync.c | 72 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 openmp/libomptarget/test/offloading/d2d_memcpy_sync.c diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp index 2dd08dd5d0b4..a0fdde951b74 100644 --- a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp +++ b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp @@ -2402,6 +2402,27 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy { AsyncInfoWrapperTy &AsyncInfoWrapper) override { AMDGPUDeviceTy &DstDevice = static_cast(DstGenericDevice); + // For large transfers use synchronous behavior. + if (Size >= OMPX_MaxAsyncCopyBytes) { + if (AsyncInfoWrapper.hasQueue()) + if (auto Err = synchronize(AsyncInfoWrapper)) + return Err; + + AMDGPUSignalTy Signal; + if (auto Err = Signal.init()) + return Err; + + if (auto Err = utils::asyncMemCopy( + useMultipleSdmaEngines(), DstPtr, DstDevice.getAgent(), SrcPtr, + getAgent(), (uint64_t)Size, 0, nullptr, Signal.get())) + return Err; + + if (auto Err = Signal.wait(getStreamBusyWaitMicroseconds())) + return Err; + + return Signal.deinit(); + } + AMDGPUStreamTy *Stream = nullptr; if (auto Err = getStream(AsyncInfoWrapper, Stream)) return Err; diff --git a/openmp/libomptarget/test/offloading/d2d_memcpy_sync.c b/openmp/libomptarget/test/offloading/d2d_memcpy_sync.c new file mode 100644 index 000000000000..6b9b765a74d8 --- /dev/null +++ b/openmp/libomptarget/test/offloading/d2d_memcpy_sync.c @@ -0,0 +1,72 @@ +// RUN: %libomptarget-compile-generic && \ +// RUN: env LIBOMPTARGET_AMDGPU_MAX_ASYNC_COPY_BYTES=0 %libomptarget-run-generic | \ +// RUN: %fcheck-generic -allow-empty +// REQUIRES: amdgcn-amd-amdhsa + +#include +#include +#include +#include + +const int magic_num = 7; + +int main(int argc, char *argv[]) { + const int N = 128; + const int num_devices = omp_get_num_devices(); + + // No target device, just return + if (num_devices == 0) { + printf("PASS\n"); + return 0; + } + + const int src_device = 0; + int dst_device = num_devices - 1; + + int length = N * sizeof(int); + int *src_ptr = omp_target_alloc(length, src_device); + int *dst_ptr = omp_target_alloc(length, dst_device); + + if (!src_ptr || !dst_ptr) { + printf("FAIL\n"); + return 1; + } + +#pragma omp target teams distribute parallel for device(src_device) \ + is_device_ptr(src_ptr) + for (int i = 0; i < N; ++i) { + src_ptr[i] = magic_num; + } + + if (omp_target_memcpy(dst_ptr, src_ptr, length, 0, 0, dst_device, + src_device)) { + printf("FAIL\n"); + return 1; + } + + int *buffer = malloc(length); + if (!buffer) { + printf("FAIL\n"); + return 1; + } + +#pragma omp target teams distribute parallel for device(dst_device) \ + map(from : buffer[0 : N]) is_device_ptr(dst_ptr) + for (int i = 0; i < N; ++i) { + buffer[i] = dst_ptr[i] + magic_num; + } + + for (int i = 0; i < N; ++i) + assert(buffer[i] == 2 * magic_num); + + printf("PASS\n"); + + // Free host and device memory + free(buffer); + omp_target_free(src_ptr, src_device); + omp_target_free(dst_ptr, dst_device); + + return 0; +} + +// CHECK: PASS -- GitLab From ddc9892999a42f9fd069f9786b82cb63312eba63 Mon Sep 17 00:00:00 2001 From: mlevesquedion Date: Fri, 29 Mar 2024 13:38:41 -0700 Subject: [PATCH 0027/1690] Add operands to worklist when only used by deleted op (#86990) I believe the existing check to determine if an operand should be added is incorrect: `operand.use_empty() || operand.hasOneUse()`. This is because these checks do not take into account the fact that the op is being deleted. It hasn't been deleted yet, so `operand.use_empty()` cannot be true, and `operand.hasOneUse()` may be true if the op being deleted is the only user of the operand and it only uses it once, but it will fail if the operand is used more than once (e.g. something like `add %0, %0`). Instead, check if the op being deleted is the only _user_ of the operand. If so, add the operand to the worklist. Fixes #86765 --- flang/test/Transforms/stack-arrays.fir | 9 +-- .../Utils/GreedyPatternRewriteDriver.cpp | 43 ++++++++++---- ...edy-pattern-rewrite-driver-bottom-up.mlir} | 0 ...reedy-pattern-rewrite-driver-top-down.mlir | 58 +++++++++++++++++++ 4 files changed, 92 insertions(+), 18 deletions(-) rename mlir/test/IR/{greedy-pattern-rewriter-driver.mlir => greedy-pattern-rewrite-driver-bottom-up.mlir} (100%) create mode 100644 mlir/test/IR/greedy-pattern-rewrite-driver-top-down.mlir diff --git a/flang/test/Transforms/stack-arrays.fir b/flang/test/Transforms/stack-arrays.fir index f4fe737e88d7..a2ffe555091e 100644 --- a/flang/test/Transforms/stack-arrays.fir +++ b/flang/test/Transforms/stack-arrays.fir @@ -127,9 +127,7 @@ func.func @placement1() { return } // CHECK: func.func @placement1() { -// CHECK-NEXT: %[[ONE:.*]] = arith.constant 1 : index -// CHECK-NEXT: %[[TWO:.*]] = arith.constant 2 : index -// CHECK-NEXT: %[[ARG:.*]] = arith.addi %[[ONE]], %[[TWO]] : index +// CHECK-NEXT: %[[ARG:.*]] = arith.constant 3 : index // CHECK-NEXT: %[[MEM:.*]] = fir.alloca !fir.array, %[[ARG]] // CHECK-NEXT: return // CHECK-NEXT: } @@ -204,13 +202,12 @@ func.func @placement4(%arg0 : i1) { // CHECK: func.func @placement4(%arg0: i1) { // CHECK-NEXT: %[[C1:.*]] = arith.constant 1 : index // CHECK-NEXT: %[[C1_I32:.*]] = fir.convert %[[C1]] : (index) -> i32 -// CHECK-NEXT: %[[C2:.*]] = arith.constant 2 : index // CHECK-NEXT: %[[C10:.*]] = arith.constant 10 : index // CHECK-NEXT: cf.br ^bb1 // CHECK-NEXT: ^bb1: -// CHECK-NEXT: %[[SUM:.*]] = arith.addi %[[C1]], %[[C2]] : index +// CHECK-NEXT: %[[C3:.*]] = arith.constant 3 : index // CHECK-NEXT: %[[SP:.*]] = fir.call @llvm.stacksave.p0() : () -> !fir.ref -// CHECK-NEXT: %[[MEM:.*]] = fir.alloca !fir.array, %[[SUM]] +// CHECK-NEXT: %[[MEM:.*]] = fir.alloca !fir.array, %[[C3]] // CHECK-NEXT: fir.call @llvm.stackrestore.p0(%[[SP]]) : (!fir.ref) -> () // CHECK-NEXT: cf.cond_br %arg0, ^bb1, ^bb2 // CHECK-NEXT: ^bb2: diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp index 6cb5635e68c9..bbecbdb85669 100644 --- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp +++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp @@ -377,7 +377,7 @@ private: /// be re-added to the worklist. This function should be called when an /// operation is modified or removed, as it may trigger further /// simplifications. - void addOperandsToWorklist(ValueRange operands); + void addOperandsToWorklist(Operation *op); /// Notify the driver that the given block was inserted. void notifyBlockInserted(Block *block, Region *previous, @@ -688,17 +688,36 @@ void GreedyPatternRewriteDriver::notifyOperationModified(Operation *op) { addToWorklist(op); } -void GreedyPatternRewriteDriver::addOperandsToWorklist(ValueRange operands) { - for (Value operand : operands) { - // If the use count of this operand is now < 2, we re-add the defining - // operation to the worklist. - // TODO: This is based on the fact that zero use operations - // may be deleted, and that single use values often have more - // canonicalization opportunities. - if (!operand || (!operand.use_empty() && !operand.hasOneUse())) +void GreedyPatternRewriteDriver::addOperandsToWorklist(Operation *op) { + for (Value operand : op->getOperands()) { + // If this operand currently has at most 2 users, add its defining op to the + // worklist. Indeed, after the op is deleted, then the operand will have at + // most 1 user left. If it has 0 users left, it can be deleted too, + // and if it has 1 user left, there may be further canonicalization + // opportunities. + if (!operand) continue; - if (auto *defOp = operand.getDefiningOp()) - addToWorklist(defOp); + + auto *defOp = operand.getDefiningOp(); + if (!defOp) + continue; + + Operation *otherUser = nullptr; + bool hasMoreThanTwoUses = false; + for (auto user : operand.getUsers()) { + if (user == op || user == otherUser) + continue; + if (!otherUser) { + otherUser = user; + continue; + } + hasMoreThanTwoUses = true; + break; + } + if (hasMoreThanTwoUses) + continue; + + addToWorklist(defOp); } } @@ -722,7 +741,7 @@ void GreedyPatternRewriteDriver::notifyOperationErased(Operation *op) { if (config.listener) config.listener->notifyOperationErased(op); - addOperandsToWorklist(op->getOperands()); + addOperandsToWorklist(op); worklist.remove(op); if (config.strictMode != GreedyRewriteStrictness::AnyOp) diff --git a/mlir/test/IR/greedy-pattern-rewriter-driver.mlir b/mlir/test/IR/greedy-pattern-rewrite-driver-bottom-up.mlir similarity index 100% rename from mlir/test/IR/greedy-pattern-rewriter-driver.mlir rename to mlir/test/IR/greedy-pattern-rewrite-driver-bottom-up.mlir diff --git a/mlir/test/IR/greedy-pattern-rewrite-driver-top-down.mlir b/mlir/test/IR/greedy-pattern-rewrite-driver-top-down.mlir new file mode 100644 index 000000000000..a362d6f99b94 --- /dev/null +++ b/mlir/test/IR/greedy-pattern-rewrite-driver-top-down.mlir @@ -0,0 +1,58 @@ +// RUN: mlir-opt %s -test-patterns="max-iterations=1 top-down=true" \ +// RUN: --split-input-file | FileCheck %s + +// Tests for https://github.com/llvm/llvm-project/issues/86765. Ensure +// that operands of a dead op are added to the worklist even if the same value +// appears multiple times as an operand. + +// 2 uses of the same operand + +// CHECK: func.func @f(%arg0: i1) { +// CHECK-NEXT: return +// CHECK-NEXT: } +func.func @f(%arg0: i1) { + %0 = arith.constant 0 : i32 + %if = scf.if %arg0 -> (i32) { + scf.yield %0 : i32 + } else { + scf.yield %0 : i32 + } + %dead_leaf = arith.addi %if, %if : i32 + return +} + +// ----- + +// 3 uses of the same operand + +// CHECK: func.func @f() { +// CHECK-NEXT: return +// CHECK-NEXT: } +func.func @f() { + %0 = arith.constant 0 : i1 + %if = scf.if %0 -> (i1) { + scf.yield %0 : i1 + } else { + scf.yield %0 : i1 + } + %dead_leaf = arith.select %if, %if, %if : i1 + return +} + +// ----- + +// 2 uses of the same operand, op has 3 operands + +// CHECK: func.func @f() { +// CHECK-NEXT: return +// CHECK-NEXT: } +func.func @f() { + %0 = arith.constant 0 : i1 + %if = scf.if %0 -> (i1) { + scf.yield %0 : i1 + } else { + scf.yield %0 : i1 + } + %dead_leaf = arith.select %0, %if, %if : i1 + return +} -- GitLab From 12fdf04ff8962c9e902669a1b600b27a960f0c11 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Fri, 29 Mar 2024 16:44:22 -0400 Subject: [PATCH 0028/1690] [clang][Darwin] Handle reexported library arguments in driver (#86980) `-reexport*` is the newer spelling for `-sub-library` which is already supported by the clang driver when invoking ld. Support the new spellings when passed by the user. This also helps simplify `clang-installapi` driver logic. --- clang/include/clang/Driver/Options.td | 3 +++ clang/test/Driver/darwin-ld-reexports.c | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 clang/test/Driver/darwin-ld-reexports.c diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 29066ea14280..8a3929eeb669 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -3635,6 +3635,9 @@ defm preserve_as_comments : BoolFOption<"preserve-as-comments", "Do not preserve comments in inline assembly">, PosFlag>; def framework : Separate<["-"], "framework">, Flags<[LinkerInput]>; +def reexport_framework : Separate<["-"], "reexport_framework">, Flags<[LinkerInput]>; +def reexport_l : Joined<["-"], "reexport-l">, Flags<[LinkerInput]>; +def reexport_library : JoinedOrSeparate<["-"], "reexport_library">, Flags<[LinkerInput]>; def frandom_seed_EQ : Joined<["-"], "frandom-seed=">, Group; def freg_struct_return : Flag<["-"], "freg-struct-return">, Group, Visibility<[ClangOption, CC1Option]>, diff --git a/clang/test/Driver/darwin-ld-reexports.c b/clang/test/Driver/darwin-ld-reexports.c new file mode 100644 index 000000000000..2e96db49a8a3 --- /dev/null +++ b/clang/test/Driver/darwin-ld-reexports.c @@ -0,0 +1,21 @@ +// RUN: touch %t.o +// RUN: %clang -target arm64-apple-darwin13 -### \ +// RUN: -reexport_framework Foo -reexport-lBar -reexport_library Baz %t.o 2> %t.log + +// Check older spellings also work. +// RUN: %clang -target arm64-apple-darwin13 -### \ +// RUN: -Xlinker -reexport_framework -Xlinker Forest \ +// RUN: -Xlinker -reexport-lBranch \ +// RUN: -Xlinker -reexport_library -Xlinker Flower %t.o 2>> %t.log +// RUN: FileCheck -check-prefix=LINK_REEXPORT %s < %t.log + +// LINK_REEXPORT: {{ld(.exe)?"}} +// LINK_REEXPORT: "-reexport_framework" "Foo" +// LINK_REEXPORT: "-reexport-lBar" +// LINK_REEXPORT: "-reexport_library" "Baz" +// LINK_REEXPORT: "-reexport_framework" "Forest" +// LINK_REEXPORT: "-reexport-lBranch" +// LINK_REEXPORT: "-reexport_library" "Flower" + +// Make sure arguments are not repeated. +// LINK_REEXPORT-NOT: "-reexport -- GitLab From cf73f136c5e22e7e4090fc762456ace47752a898 Mon Sep 17 00:00:00 2001 From: OverMighty Date: Fri, 29 Mar 2024 21:01:19 +0000 Subject: [PATCH 0029/1690] [clang][ExprConst] Fix second arg of __builtin_{clzg,ctzg} not always being evaluated (#86742) Even if we don't actually use the value of the second argument, we have to evaluate it for side-effects. --------- Co-authored-by: Richard Smith --- clang/lib/AST/ExprConstant.cpp | 30 ++++++++++------ .../constant-builtins-all-args-evaluated.cpp | 34 +++++++++++++++++++ 2 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 clang/test/Sema/constant-builtins-all-args-evaluated.cpp diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index 5a36621dc5cc..dae8f32fc029 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -12361,12 +12361,17 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E, if (!EvaluateInteger(E->getArg(0), Val, Info)) return false; + std::optional Fallback; + if (BuiltinOp == Builtin::BI__builtin_clzg && E->getNumArgs() > 1) { + APSInt FallbackTemp; + if (!EvaluateInteger(E->getArg(1), FallbackTemp, Info)) + return false; + Fallback = FallbackTemp; + } + if (!Val) { - if (BuiltinOp == Builtin::BI__builtin_clzg && E->getNumArgs() > 1) { - if (!EvaluateInteger(E->getArg(1), Val, Info)) - return false; - return Success(Val, E); - } + if (Fallback) + return Success(*Fallback, E); // When the argument is 0, the result of GCC builtins is undefined, // whereas for Microsoft intrinsics, the result is the bit-width of the @@ -12425,12 +12430,17 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E, if (!EvaluateInteger(E->getArg(0), Val, Info)) return false; + std::optional Fallback; + if (BuiltinOp == Builtin::BI__builtin_ctzg && E->getNumArgs() > 1) { + APSInt FallbackTemp; + if (!EvaluateInteger(E->getArg(1), FallbackTemp, Info)) + return false; + Fallback = FallbackTemp; + } + if (!Val) { - if (BuiltinOp == Builtin::BI__builtin_ctzg && E->getNumArgs() > 1) { - if (!EvaluateInteger(E->getArg(1), Val, Info)) - return false; - return Success(Val, E); - } + if (Fallback) + return Success(*Fallback, E); return Error(E); } diff --git a/clang/test/Sema/constant-builtins-all-args-evaluated.cpp b/clang/test/Sema/constant-builtins-all-args-evaluated.cpp new file mode 100644 index 000000000000..db55e24cecda --- /dev/null +++ b/clang/test/Sema/constant-builtins-all-args-evaluated.cpp @@ -0,0 +1,34 @@ +// RUN: %clang_cc1 -fsyntax-only -verify %s +// expected-no-diagnostics + +constexpr int test_clzg_0() { + int x = 0; + (void)__builtin_clzg(0U, ++x); + return x; +} + +static_assert(test_clzg_0() == 1); + +constexpr int test_clzg_1() { + int x = 0; + (void)__builtin_clzg(1U, ++x); + return x; +} + +static_assert(test_clzg_1() == 1); + +constexpr int test_ctzg_0() { + int x = 0; + (void)__builtin_ctzg(0U, ++x); + return x; +} + +static_assert(test_ctzg_0() == 1); + +constexpr int test_ctzg_1() { + int x = 0; + (void)__builtin_ctzg(1U, ++x); + return x; +} + +static_assert(test_ctzg_1() == 1); -- GitLab From bbb8a6c836b12968c3590a98271b30017b98ca73 Mon Sep 17 00:00:00 2001 From: Shourya Goel Date: Sat, 30 Mar 2024 02:33:35 +0530 Subject: [PATCH 0030/1690] [libc][math][c23] Fix impl and tests for X86_80 canonicalize function. (#87103) In continuation to: #87097 --- libc/src/__support/FPUtil/BasicOperations.h | 21 +++--- libc/test/src/math/smoke/CMakeLists.txt | 4 + libc/test/src/math/smoke/CanonicalizeTest.h | 84 ++++++++++----------- 3 files changed, 54 insertions(+), 55 deletions(-) diff --git a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h index a47931bb3390..6e4156497618 100644 --- a/libc/src/__support/FPUtil/BasicOperations.h +++ b/libc/src/__support/FPUtil/BasicOperations.h @@ -185,27 +185,28 @@ LIBC_INLINE int canonicalize(T &cx, const T &x) { // More precisely : // Exponent | Significand | Meaning // | Bits 63-62 | Bits 61-0 | - // All Ones | 00 | Zero | Pseudo Infinity, Value = Infinty + // All Ones | 00 | Zero | Pseudo Infinity, Value = SNaN // All Ones | 00 | Non-Zero | Pseudo NaN, Value = SNaN // All Ones | 01 | Anything | Pseudo NaN, Value = SNaN // | Bit 63 | Bits 62-0 | // All zeroes | One | Anything | Pseudo Denormal, Value = // | | | (−1)**s × m × 2**−16382 - // All Other | Zero | Anything | Unnormal, Value = - // Values | | | (−1)**s × m × 2**−16382 + // All Other | Zero | Anything | Unnormal, Value = SNaN + // Values | | | bool bit63 = sx.get_implicit_bit(); UInt128 mantissa = sx.get_explicit_mantissa(); bool bit62 = static_cast((mantissa & (1ULL << 62)) >> 62); int exponent = sx.get_biased_exponent(); if (exponent == 0x7FFF) { if (!bit63 && !bit62) { - if (mantissa == 0) - cx = FPBits::inf(sx.sign()).get_val(); - else { + if (mantissa == 0) { cx = FPBits::quiet_nan(sx.sign(), mantissa).get_val(); raise_except_if_required(FE_INVALID); return 1; } + cx = FPBits::quiet_nan(sx.sign(), mantissa).get_val(); + raise_except_if_required(FE_INVALID); + return 1; } else if (!bit63 && bit62) { cx = FPBits::quiet_nan(sx.sign(), mantissa).get_val(); raise_except_if_required(FE_INVALID); @@ -219,9 +220,11 @@ LIBC_INLINE int canonicalize(T &cx, const T &x) { cx = x; } else if (exponent == 0 && bit63) cx = FPBits::make_value(mantissa, 0).get_val(); - else if (exponent != 0 && !bit63) - cx = FPBits::make_value(mantissa, 0).get_val(); - else if (LIBC_UNLIKELY(sx.is_signaling_nan())) { + else if (exponent != 0 && !bit63) { + cx = FPBits::quiet_nan(sx.sign(), mantissa).get_val(); + raise_except_if_required(FE_INVALID); + return 1; + } else if (LIBC_UNLIKELY(sx.is_signaling_nan())) { cx = FPBits::quiet_nan(sx.sign(), sx.get_explicit_mantissa()).get_val(); raise_except_if_required(FE_INVALID); diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt index 3b756127fe21..ae2cbad7d5a7 100644 --- a/libc/test/src/math/smoke/CMakeLists.txt +++ b/libc/test/src/math/smoke/CMakeLists.txt @@ -178,6 +178,7 @@ add_fp_unittest( libc.src.math.canonicalize libc.src.__support.FPUtil.fp_bits libc.src.__support.FPUtil.fenv_impl + libc.src.__support.integer_literals ) add_fp_unittest( @@ -193,6 +194,7 @@ add_fp_unittest( libc.src.math.canonicalizef libc.src.__support.FPUtil.fp_bits libc.src.__support.FPUtil.fenv_impl + libc.src.__support.integer_literals ) add_fp_unittest( @@ -208,6 +210,7 @@ add_fp_unittest( libc.src.math.canonicalizef128 libc.src.__support.FPUtil.fp_bits libc.src.__support.FPUtil.fenv_impl + libc.src.__support.integer_literals ) add_fp_unittest( @@ -223,6 +226,7 @@ add_fp_unittest( libc.src.math.canonicalizel libc.src.__support.FPUtil.fp_bits libc.src.__support.FPUtil.fenv_impl + libc.src.__support.integer_literals ) add_fp_unittest( diff --git a/libc/test/src/math/smoke/CanonicalizeTest.h b/libc/test/src/math/smoke/CanonicalizeTest.h index c8af83f19838..4361f7d8ac7a 100644 --- a/libc/test/src/math/smoke/CanonicalizeTest.h +++ b/libc/test/src/math/smoke/CanonicalizeTest.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_CANONICALIZETEST_H #include "src/__support/FPUtil/FPBits.h" +#include "src/__support/integer_literals.h" #include "test/UnitTest/FPMatcher.h" #include "test/UnitTest/Test.h" @@ -22,6 +23,8 @@ #define TEST_REGULAR(x, y, expected) TEST_SPECIAL(x, y, expected, 0) +using LIBC_NAMESPACE::operator""_u128; + template class CanonicalizeTest : public LIBC_NAMESPACE::testing::Test { @@ -55,33 +58,31 @@ public: T cx; // Exponent | Significand | Meaning // | Bits 63-62 | Bits 61-0 | - // All Ones | 00 | Zero | Pseudo Infinity, Value = Infinty - - FPBits test1((UInt128(0x7FFF) << 64) + UInt128(0x0000000000000000)); + // All Ones | 00 | Zero | Pseudo Infinity, Value = SNaN + FPBits test1(0x00000000'00007FFF'00000000'00000000_u128); const T test1_val = test1.get_val(); - TEST_SPECIAL(cx, test1_val, 0, 0); - EXPECT_FP_EQ(cx, inf); + TEST_SPECIAL(cx, test1_val, 1, FE_INVALID); + EXPECT_FP_EQ(cx, aNaN); // Exponent | Significand | Meaning // | Bits 63-62 | Bits 61-0 | // All Ones | 00 | Non-Zero | Pseudo NaN, Value = SNaN - - FPBits test2_1((UInt128(0x7FFF) << 64) + UInt128(0x0000000000000001)); + FPBits test2_1(0x00000000'00007FFF'00000000'00000001_u128); const T test2_1_val = test2_1.get_val(); TEST_SPECIAL(cx, test2_1_val, 1, FE_INVALID); EXPECT_FP_EQ(cx, aNaN); - FPBits test2_2((UInt128(0x7FFF) << 64) + UInt128(0x0000004270000001)); + FPBits test2_2(0x00000000'00007FFF'00000042'70000001_u128); const T test2_2_val = test2_2.get_val(); TEST_SPECIAL(cx, test2_2_val, 1, FE_INVALID); EXPECT_FP_EQ(cx, aNaN); - FPBits test2_3((UInt128(0x7FFF) << 64) + UInt128(0x0000000008261001)); + FPBits test2_3(0x00000000'00007FFF'00000000'08261001_u128); const T test2_3_val = test2_3.get_val(); TEST_SPECIAL(cx, test2_3_val, 1, FE_INVALID); EXPECT_FP_EQ(cx, aNaN); - FPBits test2_4((UInt128(0x7FFF) << 64) + UInt128(0x0000780008261001)); + FPBits test2_4(0x00000000'00007FFF'00007800'08261001_u128); const T test2_4_val = test2_4.get_val(); TEST_SPECIAL(cx, test2_4_val, 1, FE_INVALID); EXPECT_FP_EQ(cx, aNaN); @@ -89,23 +90,22 @@ public: // Exponent | Significand | Meaning // | Bits 63-62 | Bits 61-0 | // All Ones | 01 | Anything | Pseudo NaN, Value = SNaN - - FPBits test3_1((UInt128(0x7FFF) << 64) + UInt128(0x4000000000000000)); + FPBits test3_1(0x00000000'00007FFF'40000000'00000000_u128); const T test3_1_val = test3_1.get_val(); TEST_SPECIAL(cx, test3_1_val, 1, FE_INVALID); EXPECT_FP_EQ(cx, aNaN); - FPBits test3_2((UInt128(0x7FFF) << 64) + UInt128(0x4000004270000001)); + FPBits test3_2(0x00000000'00007FFF'40000042'70000001_u128); const T test3_2_val = test3_2.get_val(); TEST_SPECIAL(cx, test3_2_val, 1, FE_INVALID); EXPECT_FP_EQ(cx, aNaN); - FPBits test3_3((UInt128(0x7FFF) << 64) + UInt128(0x4000000008261001)); + FPBits test3_3(0x00000000'00007FFF'40000000'08261001_u128); const T test3_3_val = test3_3.get_val(); TEST_SPECIAL(cx, test3_3_val, 1, FE_INVALID); EXPECT_FP_EQ(cx, aNaN); - FPBits test3_4((UInt128(0x7FFF) << 64) + UInt128(0x4007800008261001)); + FPBits test3_4(0x00000000'00007FFF'40007800'08261001_u128); const T test3_4_val = test3_4.get_val(); TEST_SPECIAL(cx, test3_4_val, 1, FE_INVALID); EXPECT_FP_EQ(cx, aNaN); @@ -114,20 +114,19 @@ public: // | Bit 63 | Bits 62-0 | // All zeroes | One | Anything | Pseudo Denormal, Value = // | | | (−1)**s × m × 2**−16382 - - FPBits test4_1((UInt128(0x0000) << 64) + UInt128(0x8000000000000000)); + FPBits test4_1(0x00000000'00000000'80000000'00000000_u128); const T test4_1_val = test4_1.get_val(); TEST_SPECIAL(cx, test4_1_val, 0, 0); EXPECT_FP_EQ( cx, FPBits::make_value(test4_1.get_explicit_mantissa(), 0).get_val()); - FPBits test4_2((UInt128(0x0000) << 64) + UInt128(0x8000004270000001)); + FPBits test4_2(0x00000000'00000000'80000042'70000001_u128); const T test4_2_val = test4_2.get_val(); TEST_SPECIAL(cx, test4_2_val, 0, 0); EXPECT_FP_EQ( cx, FPBits::make_value(test4_2.get_explicit_mantissa(), 0).get_val()); - FPBits test4_3((UInt128(0x0000) << 64) + UInt128(0x8000000008261001)); + FPBits test4_3(0x00000000'00000000'80000000'08261001_u128); const T test4_3_val = test4_3.get_val(); TEST_SPECIAL(cx, test4_3_val, 0, 0); EXPECT_FP_EQ( @@ -135,44 +134,37 @@ public: // Exponent | Significand | Meaning // | Bit 63 | Bits 62-0 | - // All Other | Zero | Anything | Unnormal, Value = - // Values | | | (−1)**s × m × 2**−16382 - - FPBits test5_1(UInt128(0x0000000000000001)); + // All Other | Zero | Anything | Unnormal, Value = SNaN + // Values | | | + FPBits test5_1(0x00000000'00000040'00000000'00000001_u128); const T test5_1_val = test5_1.get_val(); - TEST_SPECIAL(cx, test5_1_val, 0, 0); - EXPECT_FP_EQ( - cx, FPBits::make_value(test5_1.get_explicit_mantissa(), 0).get_val()); + TEST_SPECIAL(cx, test5_1_val, 1, FE_INVALID); + EXPECT_FP_EQ(cx, aNaN); - FPBits test5_2(UInt128(0x0000004270000001)); + FPBits test5_2(0x00000000'00000230'00000042'70000001_u128); const T test5_2_val = test5_2.get_val(); - TEST_SPECIAL(cx, test5_2_val, 0, 0); - EXPECT_FP_EQ( - cx, FPBits::make_value(test5_2.get_explicit_mantissa(), 0).get_val()); + TEST_SPECIAL(cx, test5_2_val, 1, FE_INVALID); + EXPECT_FP_EQ(cx, aNaN); - FPBits test5_3(UInt128(0x0000000008261001)); + FPBits test5_3(0x00000000'00000560'00000000'08261001_u128); const T test5_3_val = test5_3.get_val(); - TEST_SPECIAL(cx, test5_3_val, 0, 0); - EXPECT_FP_EQ( - cx, FPBits::make_value(test5_3.get_explicit_mantissa(), 0).get_val()); + TEST_SPECIAL(cx, test5_3_val, 1, FE_INVALID); + EXPECT_FP_EQ(cx, aNaN); - FPBits test5_4(UInt128(0x0000002816000000)); + FPBits test5_4(0x00000000'00000780'00000028'16000000_u128); const T test5_4_val = test5_4.get_val(); - TEST_SPECIAL(cx, test5_4_val, 0, 0); - EXPECT_FP_EQ( - cx, FPBits::make_value(test5_4.get_explicit_mantissa(), 0).get_val()); + TEST_SPECIAL(cx, test5_4_val, 1, FE_INVALID); + EXPECT_FP_EQ(cx, aNaN); - FPBits test5_5(UInt128(0x0000004270000001)); + FPBits test5_5(0x00000000'00000900'00000042'70000001_u128); const T test5_5_val = test5_5.get_val(); - TEST_SPECIAL(cx, test5_5_val, 0, 0); - EXPECT_FP_EQ( - cx, FPBits::make_value(test5_5.get_explicit_mantissa(), 0).get_val()); + TEST_SPECIAL(cx, test5_5_val, 1, FE_INVALID); + EXPECT_FP_EQ(cx, aNaN); - FPBits test5_6(UInt128(0x0000000008261001)); + FPBits test5_6(0x00000000'00000AB0'00000000'08261001_u128); const T test5_6_val = test5_6.get_val(); - TEST_SPECIAL(cx, test5_6_val, 0, 0); - EXPECT_FP_EQ( - cx, FPBits::make_value(test5_6.get_explicit_mantissa(), 0).get_val()); + TEST_SPECIAL(cx, test5_6_val, 1, FE_INVALID); + EXPECT_FP_EQ(cx, aNaN); } } -- GitLab From 13b3762608e4d3978cbb49e20f0afc3a13303f88 Mon Sep 17 00:00:00 2001 From: Diego Caballero Date: Fri, 29 Mar 2024 14:15:30 -0700 Subject: [PATCH 0031/1690] [mlir][Vector] Fix crash in drop unit dims (#87104) An `arith.select` may have a scalar condition and true/false vector values. --- .../Vector/Transforms/VectorTransforms.cpp | 10 +++-- .../vector-dropleadunitdim-transforms.mlir | 42 ++++++++++++++++++- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp index 6f6b6dcdad20..69c497264fd1 100644 --- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp @@ -1643,10 +1643,12 @@ struct DropUnitDimFromElementwiseOps final if (!resultVectorType) return failure(); - // Check the pre-conditions. For `Elementwise` Ops all operands are - // guaranteed to have identical shapes and it suffices to only check the - // first one. - auto sourceVectorType = cast(op->getOperands()[0].getType()); + // Check the operand pre-conditions. For `Elementwise` ops all operands are + // guaranteed to have identical shapes (with some exceptions such as + // `arith.select`) and it suffices to only check one of them. + auto sourceVectorType = dyn_cast(op->getOperand(0).getType()); + if (!sourceVectorType) + return failure(); if (sourceVectorType.getRank() < 2) return failure(); diff --git a/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir b/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir index 4ba51c5953d1..3a120a56056c 100644 --- a/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir @@ -40,7 +40,7 @@ func.func @cast_away_contraction_leading_one_dims(%arg0: vector<1x16x8xf32>, %ar // CHECK: %[[R1:.*]] = vector.extract %{{.*}}[0] : vector<8x16xf32> from vector<1x8x16xf32> // CHECK: %[[R2:.*]] = vector.extract %{{.*}}[0] : vector<16x16xf32> from vector<1x16x16xf32> // CHECK: %[[CONTRACT:.*]] = vector.mask %[[MASK]] { -// CHECK-SAME: vector.contract {indexing_maps = [#[[$MAP_0]], #[[$MAP_1]], #[[$MAP_2]]], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} +// CHECK-SAME: vector.contract {indexing_maps = [#[[$MAP_0]], #[[$MAP_1]], #[[$MAP_2]]], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} // CHECK-SAME: %[[R0]], %[[R1]], %[[R2]] : vector<16x8xf32>, vector<8x16xf32> into vector<16x16xf32> // CHECK-SAME: } : vector<16x16x8xi1> -> vector<16x16xf32> // CHECK: %[[RES:.*]] = vector.broadcast %[[CONTRACT]] : vector<16x16xf32> to vector<1x16x16xf32> @@ -76,7 +76,7 @@ func.func @cast_away_contraction_leading_one_dim_under_const_mask(%arg0: vector< // CHECK: %[[M:.*]] = vector.extract %{{.*}} : vector<16x16x8xi1> from vector<1x16x16x8xi1> // CHECK: %[[CONTRACT:.*]] = vector.mask %[[M]] { // CHECK-SAME: vector.contract {indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} -// CHECK-SAME: %[[R0]], %[[R1]], %[[R2]] : vector<16x8xf32>, vector<8x16xf32> into vector<16x16xf32> +// CHECK-SAME: %[[R0]], %[[R1]], %[[R2]] : vector<16x8xf32>, vector<8x16xf32> into vector<16x16xf32> // CHECK-SAME: } : vector<16x16x8xi1> -> vector<16x16xf32> // CHECK-NEXT: %[[RES:.*]] = vector.broadcast %[[CONTRACT]] : vector<16x16xf32> to vector<1x16x16xf32> // CHECK-NEXT: return %[[RES]] : vector<1x16x16xf32> @@ -472,6 +472,8 @@ func.func @cast_away_elementwise_leading_one_dims( return %0, %1, %2, %3: vector<1x1x8xf32>, vector<1x4xi1>, vector<1x4xf32>, vector<1x4xf32> } +// ----- + // CHECK-LABEL: func @cast_away_insert_leading_one_dims_scalar // CHECK-SAME: (%[[S:.+]]: f32, %[[V:.+]]: vector<1x1x4xf32>) // CHECK: %[[EXTRACT:.+]] = vector.extract %[[V]][0, 0] : vector<4xf32> from vector<1x1x4xf32> @@ -483,6 +485,8 @@ func.func @cast_away_insert_leading_one_dims_scalar(%s: f32, %v: vector<1x1x4xf3 return %0: vector<1x1x4xf32> } +// ----- + // CHECK-LABEL: func.func @cast_away_insert_leading_one_dims_scalar_scalable( // CHECK-SAME: %[[S:.*]]: f32, // CHECK-SAME: %[[V:.*]]: vector<1x1x[4]xf32>) -> vector<1x1x[4]xf32> { @@ -495,6 +499,8 @@ func.func @cast_away_insert_leading_one_dims_scalar_scalable(%s: f32, %v: vector return %0: vector<1x1x[4]xf32> } +// ----- + // CHECK-LABEL: func.func @cast_away_insert_leading_one_dims_scalar_skip_scalable_dim( // CHECK-SAME: %[[S:.*]]: f32, // CHECK-SAME: %[[V:.*]]: vector<1x[1]x4xf32>) -> vector<1x[1]x4xf32> { @@ -507,6 +513,8 @@ func.func @cast_away_insert_leading_one_dims_scalar_skip_scalable_dim(%s: f32, % return %0: vector<1x[1]x4xf32> } +// ----- + // CHECK-LABEL: func @cast_away_insert_leading_one_dims_rank1 // CHECK-SAME: (%[[S:.+]]: vector<4xf32>, %[[V:.+]]: vector<1x1x4xf32>) // CHECK: %[[BCAST:.+]] = vector.broadcast %[[S]] : vector<4xf32> to vector<1x1x4xf32> @@ -516,6 +524,8 @@ func.func @cast_away_insert_leading_one_dims_rank1(%s: vector<4xf32>, %v: vector return %0: vector<1x1x4xf32> } +// ----- + // CHECK-LABEL: func.func @cast_away_insert_leading_one_dims_rank1_scalable( // CHECK-SAME: %[[S:.*]]: vector<[4]xf32>, // CHECK-SAME: %[[V:.*]]: vector<1x1x[4]xf32>) -> vector<1x1x[4]xf32> { @@ -526,6 +536,8 @@ func.func @cast_away_insert_leading_one_dims_rank1_scalable(%s: vector<[4]xf32>, return %0: vector<1x1x[4]xf32> } +// ----- + // CHECK-LABEL: func @cast_away_insert_leading_one_dims_rank2 // CHECK-SAME: (%[[S:.+]]: vector<1x4xf32>, %[[V:.+]]: vector<1x1x4xf32>) // CHECK: %[[EXTRACT:.+]] = vector.extract %[[S]][0] : vector<4xf32> from vector<1x4xf32> @@ -536,6 +548,8 @@ func.func @cast_away_insert_leading_one_dims_rank2(%s: vector<1x4xf32>, %v: vect return %0: vector<1x1x4xf32> } +// ----- + // CHECK-LABEL: func.func @cast_away_insert_leading_one_dims_rank2_scalable( // CHECK-SAME: %[[S:.*]]: vector<1x[4]xf32>, // CHECK-SAME: %[[V:.*]]: vector<1x1x[4]xf32>) -> vector<1x1x[4]xf32> { @@ -547,6 +561,8 @@ func.func @cast_away_insert_leading_one_dims_rank2_scalable(%s: vector<1x[4]xf32 return %0: vector<1x1x[4]xf32> } +// ----- + // CHECK-LABEL: func @cast_away_insert_leading_one_dims_rank2_one_dest // CHECK-SAME: (%[[S:.+]]: vector<1x4xf32>, %[[V:.+]]: vector<1x2x1x4xf32>) // CHECK: %[[EXTRACTS:.+]] = vector.extract %[[S]][0] : vector<4xf32> from vector<1x4xf32> @@ -559,6 +575,8 @@ func.func @cast_away_insert_leading_one_dims_rank2_one_dest(%s: vector<1x4xf32>, return %0: vector<1x2x1x4xf32> } +// ----- + // CHECK-LABEL: func.func @cast_away_insert_leading_one_dims_rank2_one_dest_scalable( // CHECK-SAME: %[[S:.*]]: vector<1x[4]xf32>, // CHECK-SAME: %[[V:.*]]: vector<1x2x1x[4]xf32>) -> vector<1x2x1x[4]xf32> { @@ -572,6 +590,8 @@ func.func @cast_away_insert_leading_one_dims_rank2_one_dest_scalable(%s: vector< return %0: vector<1x2x1x[4]xf32> } +// ----- + // CHECK-LABEL: func @cast_away_insert_leading_one_dims_non_one_dest // CHECK-SAME: (%[[S:.+]]: vector<1x4xf32>, %[[V:.+]]: vector<8x1x4xf32>) // CHECK: %[[EXTRACT:.+]] = vector.extract %[[S]][0] : vector<4xf32> from vector<1x4xf32> @@ -582,6 +602,8 @@ func.func @cast_away_insert_leading_one_dims_non_one_dest(%s: vector<1x4xf32>, % return %0: vector<8x1x4xf32> } +// ----- + // CHECK-LABEL: func.func @cast_away_insert_leading_one_dims_non_one_dest_scalable( // CHECK-SAME: %[[S:.*]]: vector<1x[4]xf32>, // CHECK-SAME: %[[V:.*]]: vector<8x1x[4]xf32>) -> vector<8x1x[4]xf32> { @@ -593,6 +615,8 @@ func.func @cast_away_insert_leading_one_dims_non_one_dest_scalable(%s: vector<1x return %0: vector<8x1x[4]xf32> } +// ----- + // CHECK-LABEL: func @cast_away_insert_leading_one_dims_one_two_dest // CHECK-SAME: (%[[S:.+]]: vector<1x8xi1>, %[[V:.+]]: vector<1x1x8x1x8xi1>) // CHECK: %[[EXTRACTS:.+]] = vector.extract %[[S]][0] : vector<8xi1> from vector<1x8xi1> @@ -605,6 +629,8 @@ func.func @cast_away_insert_leading_one_dims_one_two_dest(%s: vector<1x8xi1>, %v return %0: vector<1x1x8x1x8xi1> } +// ----- + // CHECK-LABEL: func.func @cast_away_insert_leading_one_dims_one_two_dest_scalable( // CHECK-SAME: %[[S:.*]]: vector<1x[8]xi1>, // CHECK-SAME: %[[V:.*]]: vector<1x1x8x1x[8]xi1>) -> vector<1x1x8x1x[8]xi1> { @@ -618,6 +644,8 @@ func.func @cast_away_insert_leading_one_dims_one_two_dest_scalable(%s: vector<1x return %0: vector<1x1x8x1x[8]xi1> } +// ----- + // CHECK-LABEL: func.func @cast_away_constant_mask() -> vector<1x1x8x2x1xi1> { // CHECK: %[[MASK:.*]] = vector.constant_mask [6, 1, 1] : vector<8x2x1xi1> // CHECK: %[[BCAST:.*]] = vector.broadcast %[[MASK]] : vector<8x2x1xi1> to vector<1x1x8x2x1xi1> @@ -626,3 +654,13 @@ func.func @cast_away_constant_mask() -> vector<1x1x8x2x1xi1> { %0 = vector.constant_mask [1, 1, 6, 1, 1] : vector<1x1x8x2x1xi1> return %0: vector<1x1x8x2x1xi1> } + +// ----- + +// CHECK-LABEL: func.func @drop_unit_dims_scalar_cond_select( +// CHECK: arith.select {{.*}} : vector<16xi1> +func.func @drop_unit_dims_scalar_cond_select(%cond: i1, %arg0: vector<1x16xi1>, %arg1: vector<1x16xi1>) -> vector<1x16xi1> { + %sel = arith.select %cond, %arg0, %arg1 : vector<1x16xi1> + return %sel : vector<1x16xi1> +} + -- GitLab From c0a3c5c81ff7ff9e2a90b1664d6e419bf99c1aa9 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Fri, 29 Mar 2024 14:19:10 -0700 Subject: [PATCH 0032/1690] [scudo] Change tests that use setrlimit to cause mmap to fail. (#87004) It appears that qemu does not actually cause mmap to fail when calling setrlimit to limit the address space size. In the two tests that use setrlimit, detect if mmap still works and skip the tests in that case. Since all Android targets should support setrlimit, compile out the mmap check code for them. --- .../lib/scudo/standalone/tests/strings_test.cpp | 11 +++++++++++ .../lib/scudo/standalone/tests/vector_test.cpp | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp b/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp index 17a596d712d0..abb81803f65e 100644 --- a/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp @@ -129,6 +129,7 @@ TEST(ScudoStringsTest, Padding) { } #if defined(__linux__) + #include TEST(ScudoStringsTest, CapacityIncreaseFails) { @@ -136,9 +137,19 @@ TEST(ScudoStringsTest, CapacityIncreaseFails) { rlimit Limit = {}; EXPECT_EQ(0, getrlimit(RLIMIT_AS, &Limit)); + rlimit EmptyLimit = {.rlim_cur = 0, .rlim_max = Limit.rlim_max}; EXPECT_EQ(0, setrlimit(RLIMIT_AS, &EmptyLimit)); + // qemu does not honor the setrlimit, so verify before proceeding. + scudo::MemMapT MemMap; + if (MemMap.map(/*Addr=*/0U, scudo::getPageSizeCached(), "scudo:test", + MAP_ALLOWNOMEM)) { + MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); + setrlimit(RLIMIT_AS, &Limit); + GTEST_SKIP() << "Limiting address space does not prevent mmap."; + } + // Test requires that the default length is at least 6 characters. scudo::uptr MaxSize = Str.capacity(); EXPECT_LE(6u, MaxSize); diff --git a/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp b/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp index add62c5a42a3..b612676b7bd7 100644 --- a/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp @@ -58,6 +58,15 @@ TEST(ScudoVectorTest, ReallocateFails) { rlimit EmptyLimit = {.rlim_cur = 0, .rlim_max = Limit.rlim_max}; EXPECT_EQ(0, setrlimit(RLIMIT_AS, &EmptyLimit)); + // qemu does not honor the setrlimit, so verify before proceeding. + scudo::MemMapT MemMap; + if (MemMap.map(/*Addr=*/0U, scudo::getPageSizeCached(), "scudo:test", + MAP_ALLOWNOMEM)) { + MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); + setrlimit(RLIMIT_AS, &Limit); + GTEST_SKIP() << "Limiting address space does not prevent mmap."; + } + V.resize(capacity); // Set the last element so we can check it later. V.back() = '\0'; -- GitLab From 0030fc4ac74a9ce645adb9d59e108da4d4d11818 Mon Sep 17 00:00:00 2001 From: Rob Suderman Date: Fri, 29 Mar 2024 15:04:40 -0700 Subject: [PATCH 0033/1690] [mlir]Fix dialect conversion drop uses (#86991) Before deleting the block we need to drop uses to the surrounding args. If this is not performed dialect conversion failures can result in a failure to remove args (despite the block having no remaining uses). --- mlir/lib/Transforms/Utils/DialectConversion.cpp | 8 +++++--- .../TosaToLinalg/tosa-to-linalg-invalid.mlir | 13 +++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index 2ec0b964b304..3c72c8789e8e 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -279,11 +279,13 @@ public: auto &blockOps = block->getOperations(); while (!blockOps.empty()) blockOps.remove(blockOps.begin()); - block->dropAllUses(); - if (block->getParent()) + block->dropAllDefinedValueUses(); + if (block->getParent()) { block->erase(); - else + } else { + block->dropAllDefinedValueUses(); delete block; + } } }; diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir index 17eec5936918..6494e1b27194 100644 --- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir +++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir @@ -15,3 +15,16 @@ func.func @tensor_with_unknown_rank(%arg0: tensor<*xi8>) -> tensor<*xi8> { %0 = "tosa.abs"(%arg0) : (tensor<*xi8>) -> tensor<*xi8> return %0 : tensor<*xi8> } + +// ----- + +// CHECK-LABEL: @unranked_add +func.func @unranked_add(%arg0 : tensor<10x10xf32> , %arg1 : tensor<10x10xf32>, %arg2 : tensor<*xf32>) -> (tensor<10x10xf32>) { + // expected-error@+3 {{failed to legalize operation 'tosa.add'}} + %reduce = tosa.reduce_max %arg0 {axis = 1 : i32} : (tensor<10x10xf32>) -> tensor<10x1xf32> + %1 = tosa.add %reduce, %arg1 : (tensor<10x1xf32>, tensor<10x10xf32>) -> tensor<10x10xf32> + %0 = tosa.add %1, %arg2 : (tensor<10x10xf32>, tensor<*xf32>) -> tensor<*xf32> + %2 = tosa.reshape %0 {new_shape = array} : (tensor<*xf32>) -> tensor<10x10xf32> + return %2 : tensor<10x10xf32> +} + -- GitLab From 038e66fe59962de121ce24ee709eca7a0783cbc4 Mon Sep 17 00:00:00 2001 From: Jonathan Peyton Date: Fri, 29 Mar 2024 17:26:00 -0500 Subject: [PATCH 0034/1690] [OpenMP] Have hidden helper team allocate new OS threads only (#87119) The hidden helper team pre-allocates the gtid space [1, num_hidden_helpers] (inclusive). If regular host threads are allocated, then put back in the thread pool, then the hidden helper team is initialized, the hidden helper team tries to allocate the threads from the thread pool with gtids higher than [1, num_hidden_helpers]. Instead, have the hidden helper team fork OS threads so the correct gtid range used for hidden helper threads. Fixes: #87117 --- openmp/runtime/src/kmp_runtime.cpp | 8 +++-- .../tasking/hidden_helper_task/issue-87117.c | 36 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 openmp/runtime/test/tasking/hidden_helper_task/issue-87117.c diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp index a242049286a7..c715a57d23e6 100644 --- a/openmp/runtime/src/kmp_runtime.cpp +++ b/openmp/runtime/src/kmp_runtime.cpp @@ -4431,8 +4431,10 @@ kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team, #endif KMP_MB(); - /* first, try to get one from the thread pool */ - if (__kmp_thread_pool) { + /* first, try to get one from the thread pool unless allocating thread is + * the main hidden helper thread. The hidden helper team should always + * allocate new OS threads. */ + if (__kmp_thread_pool && !KMP_HIDDEN_HELPER_TEAM(team)) { new_thr = CCAST(kmp_info_t *, __kmp_thread_pool); __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool; if (new_thr == __kmp_thread_pool_insert_pt) { @@ -4497,7 +4499,7 @@ kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team, } /* no, well fork a new one */ - KMP_ASSERT(__kmp_nth == __kmp_all_nth); + KMP_ASSERT(KMP_HIDDEN_HELPER_TEAM(team) || __kmp_nth == __kmp_all_nth); KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity); #if KMP_USE_MONITOR diff --git a/openmp/runtime/test/tasking/hidden_helper_task/issue-87117.c b/openmp/runtime/test/tasking/hidden_helper_task/issue-87117.c new file mode 100644 index 000000000000..23080982f49e --- /dev/null +++ b/openmp/runtime/test/tasking/hidden_helper_task/issue-87117.c @@ -0,0 +1,36 @@ +// RUN: %libomp-compile +// RUN: env KMP_HOT_TEAMS_MODE=0 KMP_HOT_TEAMS_MAX_LEVEL=1 %libomp-run +// +// Force the defaults of: +// KMP_HOT_TEAMS_MODE=0 means free extra threads after parallel +// involving non-hot team +// KMP_HOT_TEAMS_MAX_LEVEL=1 means only the initial outer team +// is a hot team. + +#include +#include +#include + +int main() { + int a; + omp_set_max_active_levels(2); +// This nested parallel creates extra threads on the thread pool +#pragma omp parallel num_threads(2) + { +#pragma omp parallel num_threads(2) + { +#pragma omp atomic + a++; + } + } + +// Causes assert if hidden helper thread tries to allocate from thread pool +// instead of creating new OS threads +#pragma omp parallel num_threads(1) + { +#pragma omp target nowait + { a++; } + } + + return EXIT_SUCCESS; +} -- GitLab From dc4cfdbb8f9f665c1699e6289b6edfbc8d1bb443 Mon Sep 17 00:00:00 2001 From: Aart Bik Date: Fri, 29 Mar 2024 15:30:36 -0700 Subject: [PATCH 0035/1690] [mlir][sparse] provide an AoS "view" into sparse runtime support lib (#87116) Note that even though the sparse runtime support lib always uses SoA storage for COO storage (and provides correct codegen by means of views into this storage), in some rare cases we need the true physical SoA storage as a coordinate buffer. This PR provides that functionality by means of a (costly) coordinate buffer call. Since this is currently only used for testing/debugging by means of the sparse_tensor.print method, this solution is acceptable. If we ever want a performing version of this, we should truly support AoS storage of COO in addition to the SoA used right now. --- .../ExecutionEngine/SparseTensor/Storage.h | 35 ++++++++++++ .../ExecutionEngine/SparseTensorRuntime.h | 8 +++ .../Transforms/SparseTensorConversion.cpp | 56 ++++++++++++++---- .../Transforms/SparseTensorRewriting.cpp | 4 +- .../ExecutionEngine/SparseTensor/Storage.cpp | 8 +++ .../ExecutionEngine/SparseTensorRuntime.cpp | 7 +++ .../SparseTensor/CPU/sparse_print.mlir | 57 +++++++++++++++---- 7 files changed, 152 insertions(+), 23 deletions(-) diff --git a/mlir/include/mlir/ExecutionEngine/SparseTensor/Storage.h b/mlir/include/mlir/ExecutionEngine/SparseTensor/Storage.h index 773957a8b511..80e3fec22694 100644 --- a/mlir/include/mlir/ExecutionEngine/SparseTensor/Storage.h +++ b/mlir/include/mlir/ExecutionEngine/SparseTensor/Storage.h @@ -143,6 +143,12 @@ public: MLIR_SPARSETENSOR_FOREVERY_FIXED_O(DECL_GETCOORDINATES) #undef DECL_GETCOORDINATES + /// Gets coordinates-overhead storage buffer for the given level. +#define DECL_GETCOORDINATESBUFFER(INAME, C) \ + virtual void getCoordinatesBuffer(std::vector **, uint64_t); + MLIR_SPARSETENSOR_FOREVERY_FIXED_O(DECL_GETCOORDINATESBUFFER) +#undef DECL_GETCOORDINATESBUFFER + /// Gets primary storage. #define DECL_GETVALUES(VNAME, V) virtual void getValues(std::vector **); MLIR_SPARSETENSOR_FOREVERY_V(DECL_GETVALUES) @@ -251,6 +257,31 @@ public: assert(lvl < getLvlRank()); *out = &coordinates[lvl]; } + void getCoordinatesBuffer(std::vector **out, uint64_t lvl) final { + assert(out && "Received nullptr for out parameter"); + assert(lvl < getLvlRank()); + // Note that the sparse tensor support library always stores COO in SoA + // format, even when AoS is requested. This is never an issue, since all + // actual code/library generation requests "views" into the coordinate + // storage for the individual levels, which is trivially provided for + // both AoS and SoA (as well as all the other storage formats). The only + // exception is when the buffer version of coordinate storage is requested + // (currently only for printing). In that case, we do the following + // potentially expensive transformation to provide that view. If this + // operation becomes more common beyond debugging, we should consider + // implementing proper AoS in the support library as well. + uint64_t lvlRank = getLvlRank(); + uint64_t nnz = values.size(); + crdBuffer.clear(); + crdBuffer.reserve(nnz * (lvlRank - lvl)); + for (uint64_t i = 0; i < nnz; i++) { + for (uint64_t l = lvl; l < lvlRank; l++) { + assert(i < coordinates[l].size()); + crdBuffer.push_back(coordinates[l][i]); + } + } + *out = &crdBuffer; + } void getValues(std::vector **out) final { assert(out && "Received nullptr for out parameter"); *out = &values; @@ -529,10 +560,14 @@ private: return -1u; } + // Sparse tensor storage components. std::vector> positions; std::vector> coordinates; std::vector values; + + // Auxiliary data structures. std::vector lvlCursor; + std::vector crdBuffer; // just for AoS view }; //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/ExecutionEngine/SparseTensorRuntime.h b/mlir/include/mlir/ExecutionEngine/SparseTensorRuntime.h index d916186c835c..396f76fd8f92 100644 --- a/mlir/include/mlir/ExecutionEngine/SparseTensorRuntime.h +++ b/mlir/include/mlir/ExecutionEngine/SparseTensorRuntime.h @@ -77,6 +77,14 @@ MLIR_SPARSETENSOR_FOREVERY_O(DECL_SPARSEPOSITIONS) MLIR_SPARSETENSOR_FOREVERY_O(DECL_SPARSECOORDINATES) #undef DECL_SPARSECOORDINATES +/// Tensor-storage method to obtain direct access to the coordinates array +/// buffer for the given level (provides an AoS view into the library). +#define DECL_SPARSECOORDINATES(CNAME, C) \ + MLIR_CRUNNERUTILS_EXPORT void _mlir_ciface_sparseCoordinatesBuffer##CNAME( \ + StridedMemRefType *out, void *tensor, index_type lvl); +MLIR_SPARSETENSOR_FOREVERY_O(DECL_SPARSECOORDINATES) +#undef DECL_SPARSECOORDINATES + /// Tensor-storage method to insert elements in lexicographical /// level-coordinate order. #define DECL_LEXINSERT(VNAME, V) \ diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp index 92c98b34af60..c52fa3751e6b 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp @@ -275,7 +275,7 @@ static Value genPositionsCall(OpBuilder &builder, Location loc, .getResult(0); } -/// Generates a call to obtain the coordindates array. +/// Generates a call to obtain the coordinates array. static Value genCoordinatesCall(OpBuilder &builder, Location loc, SparseTensorType stt, Value ptr, Level l) { Type crdTp = stt.getCrdType(); @@ -287,6 +287,20 @@ static Value genCoordinatesCall(OpBuilder &builder, Location loc, .getResult(0); } +/// Generates a call to obtain the coordinates array (AoS view). +static Value genCoordinatesBufferCall(OpBuilder &builder, Location loc, + SparseTensorType stt, Value ptr, + Level l) { + Type crdTp = stt.getCrdType(); + auto resTp = MemRefType::get({ShapedType::kDynamic}, crdTp); + Value lvl = constantIndex(builder, loc, l); + SmallString<25> name{"sparseCoordinatesBuffer", + overheadTypeFunctionSuffix(crdTp)}; + return createFuncCall(builder, loc, name, resTp, {ptr, lvl}, + EmitCInterface::On) + .getResult(0); +} + //===----------------------------------------------------------------------===// // Conversion rules. //===----------------------------------------------------------------------===// @@ -518,13 +532,35 @@ public: LogicalResult matchAndRewrite(ToCoordinatesOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { + const Location loc = op.getLoc(); + auto stt = getSparseTensorType(op.getTensor()); + auto crds = genCoordinatesCall(rewriter, loc, stt, adaptor.getTensor(), + op.getLevel()); + // Cast the MemRef type to the type expected by the users, though these + // two types should be compatible at runtime. + if (op.getType() != crds.getType()) + crds = rewriter.create(loc, op.getType(), crds); + rewriter.replaceOp(op, crds); + return success(); + } +}; + +/// Sparse conversion rule for coordinate accesses (AoS style). +class SparseToCoordinatesBufferConverter + : public OpConversionPattern { +public: + using OpConversionPattern::OpConversionPattern; + LogicalResult + matchAndRewrite(ToCoordinatesBufferOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + const Location loc = op.getLoc(); auto stt = getSparseTensorType(op.getTensor()); - auto crds = genCoordinatesCall(rewriter, op.getLoc(), stt, - adaptor.getTensor(), op.getLevel()); + auto crds = genCoordinatesBufferCall( + rewriter, loc, stt, adaptor.getTensor(), stt.getAoSCOOStart()); // Cast the MemRef type to the type expected by the users, though these // two types should be compatible at runtime. if (op.getType() != crds.getType()) - crds = rewriter.create(op.getLoc(), op.getType(), crds); + crds = rewriter.create(loc, op.getType(), crds); rewriter.replaceOp(op, crds); return success(); } @@ -878,10 +914,10 @@ void mlir::populateSparseTensorConversionPatterns(TypeConverter &typeConverter, SparseTensorAllocConverter, SparseTensorEmptyConverter, SparseTensorDeallocConverter, SparseTensorReorderCOOConverter, SparseTensorToPositionsConverter, SparseTensorToCoordinatesConverter, - SparseTensorToValuesConverter, SparseNumberOfEntriesConverter, - SparseTensorLoadConverter, SparseTensorInsertConverter, - SparseTensorExpandConverter, SparseTensorCompressConverter, - SparseTensorAssembleConverter, SparseTensorDisassembleConverter, - SparseHasRuntimeLibraryConverter>(typeConverter, - patterns.getContext()); + SparseToCoordinatesBufferConverter, SparseTensorToValuesConverter, + SparseNumberOfEntriesConverter, SparseTensorLoadConverter, + SparseTensorInsertConverter, SparseTensorExpandConverter, + SparseTensorCompressConverter, SparseTensorAssembleConverter, + SparseTensorDisassembleConverter, SparseHasRuntimeLibraryConverter>( + typeConverter, patterns.getContext()); } diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp index 17f70d0796cc..b117c1694e45 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp @@ -648,7 +648,9 @@ public: loc, lvl, vector::PrintPunctuation::NoPunctuation); rewriter.create(loc, rewriter.getStringAttr("] : ")); Value crd = nullptr; - // TODO: eliminates ToCoordinateBufferOp! + // For COO AoS storage, we want to print a single, linear view of + // the full coordinate storage at this level. For any other storage, + // we show the coordinate storage for every indivual level. if (stt.getAoSCOOStart() == l) crd = rewriter.create(loc, tensor); else diff --git a/mlir/lib/ExecutionEngine/SparseTensor/Storage.cpp b/mlir/lib/ExecutionEngine/SparseTensor/Storage.cpp index aaa42a7e3a31..acb2d1bb5bed 100644 --- a/mlir/lib/ExecutionEngine/SparseTensor/Storage.cpp +++ b/mlir/lib/ExecutionEngine/SparseTensor/Storage.cpp @@ -68,6 +68,14 @@ MLIR_SPARSETENSOR_FOREVERY_FIXED_O(IMPL_GETPOSITIONS) MLIR_SPARSETENSOR_FOREVERY_FIXED_O(IMPL_GETCOORDINATES) #undef IMPL_GETCOORDINATES +#define IMPL_GETCOORDINATESBUFFER(CNAME, C) \ + void SparseTensorStorageBase::getCoordinatesBuffer(std::vector **, \ + uint64_t) { \ + FATAL_PIV("getCoordinatesBuffer" #CNAME); \ + } +MLIR_SPARSETENSOR_FOREVERY_FIXED_O(IMPL_GETCOORDINATESBUFFER) +#undef IMPL_GETCOORDINATESBUFFER + #define IMPL_GETVALUES(VNAME, V) \ void SparseTensorStorageBase::getValues(std::vector **) { \ FATAL_PIV("getValues" #VNAME); \ diff --git a/mlir/lib/ExecutionEngine/SparseTensorRuntime.cpp b/mlir/lib/ExecutionEngine/SparseTensorRuntime.cpp index 8835056099d2..f160b0f40fb0 100644 --- a/mlir/lib/ExecutionEngine/SparseTensorRuntime.cpp +++ b/mlir/lib/ExecutionEngine/SparseTensorRuntime.cpp @@ -311,6 +311,7 @@ MLIR_SPARSETENSOR_FOREVERY_V(IMPL_SPARSEVALUES) assert(v); \ aliasIntoMemref(v->size(), v->data(), *ref); \ } + #define IMPL_SPARSEPOSITIONS(PNAME, P) \ IMPL_GETOVERHEAD(sparsePositions##PNAME, P, getPositions) MLIR_SPARSETENSOR_FOREVERY_O(IMPL_SPARSEPOSITIONS) @@ -320,6 +321,12 @@ MLIR_SPARSETENSOR_FOREVERY_O(IMPL_SPARSEPOSITIONS) IMPL_GETOVERHEAD(sparseCoordinates##CNAME, C, getCoordinates) MLIR_SPARSETENSOR_FOREVERY_O(IMPL_SPARSECOORDINATES) #undef IMPL_SPARSECOORDINATES + +#define IMPL_SPARSECOORDINATESBUFFER(CNAME, C) \ + IMPL_GETOVERHEAD(sparseCoordinatesBuffer##CNAME, C, getCoordinatesBuffer) +MLIR_SPARSETENSOR_FOREVERY_O(IMPL_SPARSECOORDINATESBUFFER) +#undef IMPL_SPARSECOORDINATESBUFFER + #undef IMPL_GETOVERHEAD #define IMPL_LEXINSERT(VNAME, V) \ diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir index 98d76ba350cb..7758ca77dce9 100755 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir @@ -120,6 +120,14 @@ ) }> +#COOAoS = #sparse_tensor.encoding<{ + map = (d0, d1) -> (d0 : compressed(nonunique), d1 : singleton) +}> + +#COOSoA = #sparse_tensor.encoding<{ + map = (d0, d1) -> (d0 : compressed(nonunique), d1 : singleton(soa)) +}> + module { // @@ -161,6 +169,8 @@ module { %h = sparse_tensor.convert %x : tensor<4x8xi32> to tensor<4x8xi32, #BSCC> %i = sparse_tensor.convert %x : tensor<4x8xi32> to tensor<4x8xi32, #BSR0> %j = sparse_tensor.convert %x : tensor<4x8xi32> to tensor<4x8xi32, #BSC0> + %AoS = sparse_tensor.convert %x : tensor<4x8xi32> to tensor<4x8xi32, #COOAoS> + %SoA = sparse_tensor.convert %x : tensor<4x8xi32> to tensor<4x8xi32, #COOSoA> // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 @@ -274,19 +284,42 @@ module { // CHECK-NEXT: ---- sparse_tensor.print %j : tensor<4x8xi32, #BSC0> + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 5, + // CHECK-NEXT: crd[0] : ( 0, 0, 0, 2, 3, 2, 3, 3, 3, 5, + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, + // CHECK-NEXT: ---- + sparse_tensor.print %AoS : tensor<4x8xi32, #COOAoS> + + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 5, + // CHECK-NEXT: crd[0] : ( 0, 0, 3, 3, 3, + // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5, + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, + // CHECK-NEXT: ---- + sparse_tensor.print %SoA : tensor<4x8xi32, #COOSoA> + // Release the resources. - bufferization.dealloc_tensor %XO : tensor<4x8xi32, #AllDense> - bufferization.dealloc_tensor %XT : tensor<4x8xi32, #AllDenseT> - bufferization.dealloc_tensor %a : tensor<4x8xi32, #CSR> - bufferization.dealloc_tensor %b : tensor<4x8xi32, #DCSR> - bufferization.dealloc_tensor %c : tensor<4x8xi32, #CSC> - bufferization.dealloc_tensor %d : tensor<4x8xi32, #DCSC> - bufferization.dealloc_tensor %e : tensor<4x8xi32, #BSR> - bufferization.dealloc_tensor %f : tensor<4x8xi32, #BSRC> - bufferization.dealloc_tensor %g : tensor<4x8xi32, #BSC> - bufferization.dealloc_tensor %h : tensor<4x8xi32, #BSCC> - bufferization.dealloc_tensor %i : tensor<4x8xi32, #BSR0> - bufferization.dealloc_tensor %j : tensor<4x8xi32, #BSC0> + bufferization.dealloc_tensor %XO : tensor<4x8xi32, #AllDense> + bufferization.dealloc_tensor %XT : tensor<4x8xi32, #AllDenseT> + bufferization.dealloc_tensor %a : tensor<4x8xi32, #CSR> + bufferization.dealloc_tensor %b : tensor<4x8xi32, #DCSR> + bufferization.dealloc_tensor %c : tensor<4x8xi32, #CSC> + bufferization.dealloc_tensor %d : tensor<4x8xi32, #DCSC> + bufferization.dealloc_tensor %e : tensor<4x8xi32, #BSR> + bufferization.dealloc_tensor %f : tensor<4x8xi32, #BSRC> + bufferization.dealloc_tensor %g : tensor<4x8xi32, #BSC> + bufferization.dealloc_tensor %h : tensor<4x8xi32, #BSCC> + bufferization.dealloc_tensor %i : tensor<4x8xi32, #BSR0> + bufferization.dealloc_tensor %j : tensor<4x8xi32, #BSC0> + bufferization.dealloc_tensor %AoS : tensor<4x8xi32, #COOAoS> + bufferization.dealloc_tensor %SoA : tensor<4x8xi32, #COOSoA> return } -- GitLab From b4693e0d7da8f7f1f1365ab97ecc0e33ad459943 Mon Sep 17 00:00:00 2001 From: Vladimir Vereschaka Date: Fri, 29 Mar 2024 15:31:31 -0700 Subject: [PATCH 0036/1690] [CMake] Update CMake cache file for the Win-to-Arm cross toolchains. NFC. (#87113) Review the actual component parameters and update the cache file accordingly. Also fixed the C++ test builds for the compiler-rt component. --- clang/cmake/caches/CrossWinToARMLinux.cmake | 42 ++++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/clang/cmake/caches/CrossWinToARMLinux.cmake b/clang/cmake/caches/CrossWinToARMLinux.cmake index 2a0953af53fa..736a54ece550 100644 --- a/clang/cmake/caches/CrossWinToARMLinux.cmake +++ b/clang/cmake/caches/CrossWinToARMLinux.cmake @@ -29,6 +29,11 @@ # cmake --build . --target check-cxxabi- # cmake --build . --target check-unwind- # cmake --build . --target check-cxx- +# (another way to execute the tests) +# python bin/llvm-lit.py -v --threads=32 runtimes/runtimes-bins/libunwind/test 2>&1 | tee libunwind-tests.log +# python bin/llvm-lit.py -v --threads=32 runtimes/runtimes--bins/libcxxabi/test 2>&1 | tee libcxxabi-tests.log +# python bin/llvm-lit.py -v --threads=32 runtimes/runtimes--bins/libcxx/test 2>&1 | tee libcxx-tests.log + # LLVM_PROJECT_DIR is the path to the llvm-project directory. # The right way to compute it would probably be to use "${CMAKE_SOURCE_DIR}/../", @@ -42,9 +47,6 @@ if (NOT DEFINED DEFAULT_SYSROOT) message(WARNING "DEFAULT_SYSROOT must be specified for the cross toolchain build.") endif() -if (NOT DEFINED LLVM_TARGETS_TO_BUILD) - set(LLVM_TARGETS_TO_BUILD "ARM" CACHE STRING "") -endif() if (NOT DEFINED LLVM_ENABLE_ASSERTIONS) set(LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "") endif() @@ -56,7 +58,7 @@ if (NOT DEFINED LLVM_ENABLE_RUNTIMES) endif() if (NOT DEFINED TOOLCHAIN_TARGET_TRIPLE) - set(TOOLCHAIN_TARGET_TRIPLE "armv7-unknown-linux-gnueabihf") + set(TOOLCHAIN_TARGET_TRIPLE "aarch64-unknown-linux-gnu") else() #NOTE: we must normalize specified target triple to a fully specified triple, # including the vendor part. It is necessary to synchronize the runtime library @@ -74,24 +76,38 @@ else() string(REPLACE ";" "-" TOOLCHAIN_TARGET_TRIPLE "${TOOLCHAIN_TARGET_TRIPLE}") endif() +message(STATUS "Toolchain target triple: ${TOOLCHAIN_TARGET_TRIPLE}") + +if (NOT DEFINED LLVM_TARGETS_TO_BUILD) + if ("${TOOLCHAIN_TARGET_TRIPLE}" MATCHES "^(armv|arm32)+") + set(LLVM_TARGETS_TO_BUILD "ARM" CACHE STRING "") + endif() + if ("${TOOLCHAIN_TARGET_TRIPLE}" MATCHES "^(aarch64|arm64)+") + set(LLVM_TARGETS_TO_BUILD "AArch64" CACHE STRING "") + endif() +endif() + +message(STATUS "Toolchain target to build: ${LLVM_TARGETS_TO_BUILD}") + if (NOT DEFINED CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") endif() -message(STATUS "Toolchain target triple: ${TOOLCHAIN_TARGET_TRIPLE}") - set(CMAKE_CROSSCOMPILING ON CACHE BOOL "") set(CMAKE_CL_SHOWINCLUDES_PREFIX "Note: including file: " CACHE STRING "") # Required if COMPILER_RT_DEFAULT_TARGET_ONLY is ON set(CMAKE_C_COMPILER_TARGET "${TOOLCHAIN_TARGET_TRIPLE}" CACHE STRING "") set(CMAKE_CXX_COMPILER_TARGET "${TOOLCHAIN_TARGET_TRIPLE}" CACHE STRING "") -set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ON CACHE BOOL "") set(LLVM_DEFAULT_TARGET_TRIPLE "${TOOLCHAIN_TARGET_TRIPLE}" CACHE STRING "") set(LLVM_TARGET_ARCH "${TOOLCHAIN_TARGET_TRIPLE}" CACHE STRING "") set(LLVM_LIT_ARGS "-vv ${LLVM_LIT_ARGS}" CACHE STRING "" FORCE) +set(CLANG_DEFAULT_CXX_STDLIB "libc++" CACHE STRING "") set(CLANG_DEFAULT_LINKER "lld" CACHE STRING "") +set(CLANG_DEFAULT_OBJCOPY "llvm-objcopy" CACHE STRING "") +set(CLANG_DEFAULT_RTLIB "compiler-rt" CACHE STRING "") +set(CLANG_DEFAULT_UNWINDLIB "libunwind" CACHE STRING "") if(WIN32) set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded" CACHE STRING "") @@ -109,9 +125,10 @@ set(BUILTINS_${TOOLCHAIN_TARGET_TRIPLE}_CMAKE_SYSTEM_NAME set(BUILTINS_${TOOLCHAIN_TARGET_TRIPLE}_CMAKE_SYSROOT "${DEFAULT_SYSROOT}" CACHE STRING "") set(BUILTINS_${TOOLCHAIN_TARGET_TRIPLE}_CMAKE_INSTALL_RPATH "${RUNTIMES_INSTALL_RPATH}" CACHE STRING "") set(BUILTINS_${TOOLCHAIN_TARGET_TRIPLE}_CMAKE_BUILD_WITH_INSTALL_RPATH ON CACHE BOOL "") - +set(BUILTINS_${TOOLCHAIN_TARGET_TRIPLE}_LLVM_CMAKE_DIR "${LLVM_PROJECT_DIR}/llvm/cmake/modules" CACHE PATH "") set(LLVM_RUNTIME_TARGETS "${TOOLCHAIN_TARGET_TRIPLE}" CACHE STRING "") +set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ON CACHE BOOL "") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LLVM_ENABLE_RUNTIMES "${LLVM_ENABLE_RUNTIMES}" CACHE STRING "") @@ -125,13 +142,16 @@ set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_BUILD_SANITIZERS set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_BUILD_XRAY OFF CACHE BOOL "") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_BUILD_LIBFUZZER OFF CACHE BOOL "") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_BUILD_PROFILE OFF CACHE BOOL "") -set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_BUILD_CRT OFF CACHE BOOL "") +set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_BUILD_CRT ON CACHE BOOL "") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_BUILD_ORC OFF CACHE BOOL "") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_DEFAULT_TARGET_ONLY ON CACHE BOOL "") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_INCLUDE_TESTS ON CACHE BOOL "") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_CAN_EXECUTE_TESTS ON CACHE BOOL "") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "") +set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_CXX_LIBRARY libcxx CACHE STRING "") +# Tell Clang to seach C++ headers alongside with the just-built binaries for the C++ compiler-rt tests. +set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_TEST_COMPILER_CFLAGS "--stdlib=libc++" CACHE STRING "") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBUNWIND_USE_COMPILER_RT ON CACHE BOOL "") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBUNWIND_ENABLE_SHARED OFF CACHE BOOL "") @@ -148,8 +168,10 @@ set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_ABI_VERSION set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_CXX_ABI "libcxxabi" CACHE STRING "") #!!! set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_ENABLE_NEW_DELETE_DEFINITIONS ON CACHE BOOL "") - +# Avoid searching for the python3 interpreter during the runtimes configuration for the cross builds. +# It starts searching the python3 package using the target's sysroot path, that usually is not compatible with the build host. find_package(Python3 COMPONENTS Interpreter) +set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_Python3_EXECUTABLE ${Python3_EXECUTABLE} CACHE PATH "") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBUNWIND_TEST_PARAMS_default "${RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_TEST_PARAMS}") set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_TEST_PARAMS_default "${RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_TEST_PARAMS}") -- GitLab From 7467dc188a0c63b32bd3068fc98ae77f86285c0d Mon Sep 17 00:00:00 2001 From: Sitnikov Sergey <109442154+Aqendo@users.noreply.github.com> Date: Fri, 29 Mar 2024 15:43:01 -0700 Subject: [PATCH 0037/1690] Fix apostrophes in assert error message (#69881) --- openmp/libomptarget/DeviceRTL/src/Debug.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openmp/libomptarget/DeviceRTL/src/Debug.cpp b/openmp/libomptarget/DeviceRTL/src/Debug.cpp index aecc33c0497a..31cd54e3de35 100644 --- a/openmp/libomptarget/DeviceRTL/src/Debug.cpp +++ b/openmp/libomptarget/DeviceRTL/src/Debug.cpp @@ -33,10 +33,10 @@ void __assert_fail(const char *expr, const char *file, unsigned line, void __assert_fail_internal(const char *expr, const char *msg, const char *file, unsigned line, const char *function) { if (msg) { - PRINTF("%s:%u: %s: Assertion %s (`%s') failed.\n", file, line, function, + PRINTF("%s:%u: %s: Assertion %s (`%s`) failed.\n", file, line, function, msg, expr); } else { - PRINTF("%s:%u: %s: Assertion `%s' failed.\n", file, line, function, expr); + PRINTF("%s:%u: %s: Assertion `%s` failed.\n", file, line, function, expr); } __builtin_trap(); } -- GitLab From 7de82ca369b5e9114636420a16e1c5d25283421d Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Fri, 29 Mar 2024 16:41:15 -0700 Subject: [PATCH 0038/1690] [BOLT] Don't terminate on trap instruction for Linux kernel (#87021) Under normal circumstances, we terminate basic blocks on a trap instruction. However, Linux kernel may resume execution after hitting a trap (ud2 on x86). Thus, we introduce "--terminal-trap" option that will specify if the trap instruction should terminate the control flow. The option is on by default except for the Linux kernel mode when it's off. --- bolt/include/bolt/Core/MCPlusBuilder.h | 4 +--- bolt/lib/Core/MCPlusBuilder.cpp | 14 ++++++++++++++ bolt/lib/Rewrite/RewriteInstance.cpp | 11 +++++++++-- bolt/lib/Target/X86/X86MCPlusBuilder.cpp | 7 ------- bolt/test/X86/linux-bug-table.s | 4 ++++ 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h index 198a8d8bf48f..3818e3d66c99 100644 --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -533,9 +533,7 @@ public: return Analysis->isReturn(Inst); } - virtual bool isTerminator(const MCInst &Inst) const { - return Analysis->isTerminator(Inst); - } + virtual bool isTerminator(const MCInst &Inst) const; virtual bool isNoop(const MCInst &Inst) const { llvm_unreachable("not implemented"); diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp index 5b14ad5cdb88..f22b35a90bc5 100644 --- a/bolt/lib/Core/MCPlusBuilder.cpp +++ b/bolt/lib/Core/MCPlusBuilder.cpp @@ -12,12 +12,14 @@ #include "bolt/Core/MCPlusBuilder.h" #include "bolt/Core/MCPlus.h" +#include "bolt/Utils/CommandLineOpts.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstrAnalysis.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include #include @@ -28,6 +30,13 @@ using namespace llvm; using namespace bolt; using namespace MCPlus; +namespace opts { +cl::opt + TerminalTrap("terminal-trap", + cl::desc("Assume that execution stops at trap instruction"), + cl::init(true), cl::Hidden, cl::cat(BoltCategory)); +} + bool MCPlusBuilder::equals(const MCInst &A, const MCInst &B, CompFuncTy Comp) const { if (A.getOpcode() != B.getOpcode()) @@ -121,6 +130,11 @@ bool MCPlusBuilder::equals(const MCTargetExpr &A, const MCTargetExpr &B, llvm_unreachable("target-specific expressions are unsupported"); } +bool MCPlusBuilder::isTerminator(const MCInst &Inst) const { + return Analysis->isTerminator(Inst) || + (opts::TerminalTrap && Info->get(Inst.getOpcode()).isTrap()); +} + void MCPlusBuilder::setTailCall(MCInst &Inst) const { assert(!hasAnnotation(Inst, MCAnnotation::kTailCall)); setAnnotationOpValue(Inst, MCAnnotation::kTailCall, true); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 2ead51ff6a12..5ca5594117a6 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -84,6 +84,7 @@ extern cl::opt JumpTables; extern cl::opt KeepNops; extern cl::list ReorderData; extern cl::opt ReorderFunctions; +extern cl::opt TerminalTrap; extern cl::opt TimeBuild; cl::opt AllowStripped("allow-stripped", @@ -2033,8 +2034,14 @@ void RewriteInstance::adjustCommandLineOptions() { if (opts::Lite) BC->outs() << "BOLT-INFO: enabling lite mode\n"; - if (BC->IsLinuxKernel && !opts::KeepNops.getNumOccurrences()) - opts::KeepNops = true; + if (BC->IsLinuxKernel) { + if (!opts::KeepNops.getNumOccurrences()) + opts::KeepNops = true; + + // Linux kernel may resume execution after a trap instruction in some cases. + if (!opts::TerminalTrap.getNumOccurrences()) + opts::TerminalTrap = false; + } } namespace { diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp index 15f95f821777..8b1894953f37 100644 --- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp +++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp @@ -211,13 +211,6 @@ public: return false; } - // FIXME: For compatibility with old LLVM only! - bool isTerminator(const MCInst &Inst) const override { - unsigned Opcode = Inst.getOpcode(); - return Info->get(Opcode).isTerminator() || X86::isUD1(Opcode) || - X86::isUD2(Opcode); - } - bool isIndirectCall(const MCInst &Inst) const override { return isCall(Inst) && ((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(0).isReg()); diff --git a/bolt/test/X86/linux-bug-table.s b/bolt/test/X86/linux-bug-table.s index f688a60c9771..63f70a0b35d9 100644 --- a/bolt/test/X86/linux-bug-table.s +++ b/bolt/test/X86/linux-bug-table.s @@ -40,6 +40,10 @@ _start: # CHECK-REOPT-SAME: BugEntry: 2 ret +## The return instruction is reachable only via preceding ud2. Test that it is +## treated as a reachable instruction in the Linux kernel mode. + +# CHECK-REOPT-NEXT: ret .size _start, .-_start -- GitLab From a8b0ecd2605ff23f495a8af64e06c35f86834e54 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Fri, 29 Mar 2024 20:17:11 -0400 Subject: [PATCH 0039/1690] [libc] enable stack protectors and frame pointers on default (#86288) --- libc/cmake/modules/LLVMLibCCompileOptionRules.cmake | 9 +++++++++ libc/config/config.json | 10 ++++++++++ libc/docs/configure.rst | 3 +++ .../llvm-project-overlay/libc/libc_build_rules.bzl | 9 ++++++++- 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake index 5bc0898298ce..40a1cfda060e 100644 --- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake +++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake @@ -60,6 +60,15 @@ function(_get_common_compile_options output_var flags) if (LIBC_CC_SUPPORTS_PATTERN_INIT) list(APPEND compile_options "-ftrivial-auto-var-init=pattern") endif() + if (LIBC_CONF_KEEP_FRAME_POINTER) + list(APPEND compile_options "-fno-omit-frame-pointer") + if (LIBC_TARGET_ARCHITECTURE_IS_X86) + list(APPEND compile_options "-mno-omit-leaf-frame-pointer") + endif() + endif() + if (LIBC_CONF_ENABLE_STACK_PROTECTOR) + list(APPEND compile_options "-fstack-protector-strong") + endif() list(APPEND compile_options "-Wall") list(APPEND compile_options "-Wextra") # -DLIBC_WNO_ERROR=ON if you can't build cleanly with -Werror. diff --git a/libc/config/config.json b/libc/config/config.json index b73c47b1a14b..d6ef891b9f26 100644 --- a/libc/config/config.json +++ b/libc/config/config.json @@ -30,5 +30,15 @@ "value": false, "doc": "Inserts prefetch for write instructions (PREFETCHW) for memset on x86 to recover performance when hardware prefetcher is disabled." } + }, + "codegen": { + "LIBC_CONF_KEEP_FRAME_POINTER": { + "value": true, + "doc": "Keep frame pointer in functions for better debugging experience." + }, + "LIBC_CONF_ENABLE_STRONG_STACK_PROTECTOR": { + "value": true, + "doc": "Enable -fstack-protector-strong to defend against stack smashing attack." + } } } diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst index a177550647bd..8f8c44caa115 100644 --- a/libc/docs/configure.rst +++ b/libc/docs/configure.rst @@ -25,6 +25,9 @@ See the main ``config/config.json``, and the platform and architecture specific overrides in ``config//config.json`` and ``config///config.json,`` to learn about the defaults for your platform and target. +* **"codegen" options** + - ``LIBC_CONF_ENABLE_STRONG_STACK_PROTECTOR``: Enable -fstack-protector-strong to defend against stack smashing attack. + - ``LIBC_CONF_KEEP_FRAME_POINTER``: Keep frame pointer in functions for better debugging experience. * **"printf" options** - ``LIBC_CONF_PRINTF_DISABLE_FIXED_POINT``: Disable printing fixed point values in printf and friends. - ``LIBC_CONF_PRINTF_DISABLE_FLOAT``: Disable printing floating point values in printf and friends. diff --git a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl index 7d815bc4a229..7dc12bade260 100644 --- a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl +++ b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl @@ -78,7 +78,6 @@ def libc_function( its deps. **kwargs: Other attributes relevant for a cc_library. For example, deps. """ - # We use the explicit equals pattern here because append and += mutate the # original list, where this creates a new list and stores it in deps. copts = copts or [] @@ -87,7 +86,15 @@ def libc_function( "-fno-builtin", "-fno-lax-vector-conversions", "-ftrivial-auto-var-init=pattern", + "-fno-omit-frame-pointer", + "-fstack-protector-strong", ] + # x86 targets have -mno-omit-leaf-frame-pointer. + platform_copts = selects.with_or({ + PLATFORM_CPU_X86_64: ["-mno-omit-leaf-frame-pointer"], + "//conditions:default": [] + }) + copts = copts + platform_copts # We compile the code twice, the first target is suffixed with ".__internal__" and contains the # C++ functions in the "LIBC_NAMESPACE" namespace. This allows us to test the function in the -- GitLab From 5538853f9abb559ff4bb7ff598535041d91d97d7 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 30 Mar 2024 09:54:58 +0800 Subject: [PATCH 0040/1690] [PassManager] Support MachineFunctionProperties (#83668) This pull request adds `MachineFunctionProperties` support. If a pass wants to modify machine function properties, it must derive from `MachinePassInfoMixin` and define some static methods like in legacy pass manager. A test pass `RequireAllMachineFunctionPropertiesPass` is also added here, which could be a example. --- .../include/llvm/CodeGen/MachinePassManager.h | 187 ++++++++---------- .../llvm/Passes/MachinePassRegistry.def | 2 + llvm/lib/Passes/PassBuilder.cpp | 27 +++ .../new-pm/machine-function-properties.mir | 12 ++ 4 files changed, 125 insertions(+), 103 deletions(-) create mode 100644 llvm/test/tools/llc/new-pm/machine-function-properties.mir diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h index 3faffe5c4cab..965e8282ccab 100644 --- a/llvm/include/llvm/CodeGen/MachinePassManager.h +++ b/llvm/include/llvm/CodeGen/MachinePassManager.h @@ -16,8 +16,6 @@ // their respective analysis managers such as ModuleAnalysisManager and // FunctionAnalysisManager. // -// TODO: Add MachineFunctionProperties support. -// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_MACHINEPASSMANAGER_H @@ -44,23 +42,67 @@ using MachineFunctionAnalysisManager = AnalysisManager; /// automatically mixes in \c PassInfoMixin. template struct MachinePassInfoMixin : public PassInfoMixin { - // TODO: Add MachineFunctionProperties support. +protected: + class PropertyChanger { + MachineFunction &MF; + + template + using has_get_required_properties_t = + decltype(std::declval().getRequiredProperties()); + + template + using has_get_set_properties_t = + decltype(std::declval().getSetProperties()); + + template + using has_get_cleared_properties_t = + decltype(std::declval().getClearedProperties()); + + public: + PropertyChanger(MachineFunction &MF) : MF(MF) { +#ifndef NDEBUG + if constexpr (is_detected::value) { + auto &MFProps = MF.getProperties(); + auto RequiredProperties = DerivedT::getRequiredProperties(); + if (!MFProps.verifyRequiredProperties(RequiredProperties)) { + errs() << "MachineFunctionProperties required by " << DerivedT::name() + << " pass are not met by function " << MF.getName() << ".\n" + << "Required properties: "; + RequiredProperties.print(errs()); + errs() << "\nCurrent properties: "; + MFProps.print(errs()); + errs() << '\n'; + report_fatal_error("MachineFunctionProperties check failed"); + } +#endif + } + } + + ~PropertyChanger() { + if constexpr (is_detected::value) + MF.getProperties().set(DerivedT::getSetProperties()); + if constexpr (is_detected::value) + MF.getProperties().reset(DerivedT::getClearedProperties()); + } + }; + +public: + PreservedAnalyses runImpl(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM) { + PropertyChanger PC(MF); + return static_cast(this)->run(MF, MFAM); + } }; namespace detail { -struct MachinePassConcept - : PassConcept { - virtual MachineFunctionProperties getRequiredProperties() const = 0; - virtual MachineFunctionProperties getSetProperties() const = 0; - virtual MachineFunctionProperties getClearedProperties() const = 0; -}; -template struct MachinePassModel : MachinePassConcept { - explicit MachinePassModel(PassT &&Pass) : Pass(std::move(Pass)) {} - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - MachinePassModel(const MachinePassModel &Arg) : Pass(Arg.Pass) {} - MachinePassModel(MachinePassModel &&Arg) : Pass(std::move(Arg.Pass)) {} +template +struct MachinePassModel + : PassModel { + explicit MachinePassModel(PassT &&Pass) + : PassModel( + std::move(Pass)) {} friend void swap(MachinePassModel &LHS, MachinePassModel &RHS) { using std::swap; @@ -75,89 +117,8 @@ template struct MachinePassModel : MachinePassConcept { MachinePassModel &operator=(const MachinePassModel &) = delete; PreservedAnalyses run(MachineFunction &IR, MachineFunctionAnalysisManager &AM) override { - return Pass.run(IR, AM); - } - - void printPipeline( - raw_ostream &OS, - function_ref MapClassName2PassName) override { - Pass.printPipeline(OS, MapClassName2PassName); - } - - StringRef name() const override { return PassT::name(); } - - template - using has_required_t = decltype(std::declval().isRequired()); - template - static std::enable_if_t::value, bool> - passIsRequiredImpl() { - return T::isRequired(); + return this->Pass.runImpl(IR, AM); } - template - static std::enable_if_t::value, bool> - passIsRequiredImpl() { - return false; - } - bool isRequired() const override { return passIsRequiredImpl(); } - - template - using has_get_required_properties_t = - decltype(std::declval().getRequiredProperties()); - template - static std::enable_if_t::value, - MachineFunctionProperties> - getRequiredPropertiesImpl() { - return PassT::getRequiredProperties(); - } - template - static std::enable_if_t::value, - MachineFunctionProperties> - getRequiredPropertiesImpl() { - return MachineFunctionProperties(); - } - MachineFunctionProperties getRequiredProperties() const override { - return getRequiredPropertiesImpl(); - } - - template - using has_get_set_properties_t = - decltype(std::declval().getSetProperties()); - template - static std::enable_if_t::value, - MachineFunctionProperties> - getSetPropertiesImpl() { - return PassT::getSetProperties(); - } - template - static std::enable_if_t::value, - MachineFunctionProperties> - getSetPropertiesImpl() { - return MachineFunctionProperties(); - } - MachineFunctionProperties getSetProperties() const override { - return getSetPropertiesImpl(); - } - - template - using has_get_cleared_properties_t = - decltype(std::declval().getClearedProperties()); - template - static std::enable_if_t::value, - MachineFunctionProperties> - getClearedPropertiesImpl() { - return PassT::getClearedProperties(); - } - template - static std::enable_if_t::value, - MachineFunctionProperties> - getClearedPropertiesImpl() { - return MachineFunctionProperties(); - } - MachineFunctionProperties getClearedProperties() const override { - return getClearedPropertiesImpl(); - } - - PassT Pass; }; } // namespace detail @@ -251,11 +212,12 @@ private: class ModuleToMachineFunctionPassAdaptor : public PassInfoMixin { - using MachinePassConcept = detail::MachinePassConcept; - public: + using PassConceptT = + detail::PassConcept; + explicit ModuleToMachineFunctionPassAdaptor( - std::unique_ptr Pass) + std::unique_ptr Pass) : Pass(std::move(Pass)) {} /// Runs the function pass across every function in the module. @@ -266,20 +228,39 @@ public: static bool isRequired() { return true; } private: - std::unique_ptr Pass; + std::unique_ptr Pass; }; template ModuleToMachineFunctionPassAdaptor createModuleToMachineFunctionPassAdaptor(MachineFunctionPassT &&Pass) { - using PassModelT = detail::MachinePassModel; + using PassModelT = detail::PassModel; // Do not use make_unique, it causes too many template instantiations, // causing terrible compile times. return ModuleToMachineFunctionPassAdaptor( - std::unique_ptr( + std::unique_ptr( new PassModelT(std::forward(Pass)))); } +template <> +template +std::enable_if_t>::value> +PassManager::addPass(PassT &&Pass) { + using PassModelT = + detail::PassModel; + using MachinePassModelT = detail::MachinePassModel; + // Do not use make_unique or emplace_back, they cause too many template + // instantiations, causing terrible compile times. + if constexpr (std::is_base_of_v, PassT>) { + Passes.push_back(std::unique_ptr( + new MachinePassModelT(std::forward(Pass)))); + } else { + Passes.push_back(std::unique_ptr( + new PassModelT(std::forward(Pass)))); + } +} + template <> PreservedAnalyses PassManager::run(MachineFunction &, diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index 016602730e0e..2f77ae655d9b 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -127,6 +127,8 @@ MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass()) // MACHINE_FUNCTION_PASS("free-machine-function", FreeMachineFunctionPass()) MACHINE_FUNCTION_PASS("no-op-machine-function", NoOpMachineFunctionPass()) MACHINE_FUNCTION_PASS("print", PrintMIRPass()) +MACHINE_FUNCTION_PASS("require-all-machine-function-properties", + RequireAllMachineFunctionPropertiesPass()) #undef MACHINE_FUNCTION_PASS // After a pass is converted to new pass manager, its entry should be moved from diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index f60f4eb3f0ef..57975e34d426 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -365,6 +365,33 @@ public: static StringRef name() { return "TriggerVerifierErrorPass"; } }; +// A pass requires all MachineFunctionProperties. +// DO NOT USE THIS EXCEPT FOR TESTING! +class RequireAllMachineFunctionPropertiesPass + : public MachinePassInfoMixin { +public: + PreservedAnalyses run(MachineFunction &, MachineFunctionAnalysisManager &) { + return PreservedAnalyses::none(); + } + + static MachineFunctionProperties getRequiredProperties() { + MachineFunctionProperties MFProps; + MFProps.set(MachineFunctionProperties::Property::FailedISel); + MFProps.set(MachineFunctionProperties::Property::FailsVerification); + MFProps.set(MachineFunctionProperties::Property::IsSSA); + MFProps.set(MachineFunctionProperties::Property::Legalized); + MFProps.set(MachineFunctionProperties::Property::NoPHIs); + MFProps.set(MachineFunctionProperties::Property::NoVRegs); + MFProps.set(MachineFunctionProperties::Property::RegBankSelected); + MFProps.set(MachineFunctionProperties::Property::Selected); + MFProps.set(MachineFunctionProperties::Property::TiedOpsRewritten); + MFProps.set(MachineFunctionProperties::Property::TracksDebugUserValues); + MFProps.set(MachineFunctionProperties::Property::TracksLiveness); + return MFProps; + } + static StringRef name() { return "RequireAllMachineFunctionPropertiesPass"; } +}; + } // namespace PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO, diff --git a/llvm/test/tools/llc/new-pm/machine-function-properties.mir b/llvm/test/tools/llc/new-pm/machine-function-properties.mir new file mode 100644 index 000000000000..a9eb88ec6988 --- /dev/null +++ b/llvm/test/tools/llc/new-pm/machine-function-properties.mir @@ -0,0 +1,12 @@ +# REQUIRES: asserts +# RUN: not --crash llc -mtriple=x86_64-pc-linux-gnu -passes=require-all-machine-function-properties -filetype=null %s 2>&1 | FileCheck %s + +# CHECK: MachineFunctionProperties required by RequireAllMachineFunctionPropertiesPass pass are not met by function f. + +--- +name: f +selected: false +body: | + bb.0: + RET 0 +... -- GitLab From 0be26dbbed91ba84b4d00bd2e19e2f60aeb51a9f Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 30 Mar 2024 10:02:49 +0800 Subject: [PATCH 0041/1690] Revert "[PassManager] Support MachineFunctionProperties (#83668)" (#87136) This reverts commit 5538853f9abb559ff4bb7ff598535041d91d97d7. #83668 Break some test bots. --- .../include/llvm/CodeGen/MachinePassManager.h | 187 ++++++++++-------- .../llvm/Passes/MachinePassRegistry.def | 2 - llvm/lib/Passes/PassBuilder.cpp | 27 --- .../new-pm/machine-function-properties.mir | 12 -- 4 files changed, 103 insertions(+), 125 deletions(-) delete mode 100644 llvm/test/tools/llc/new-pm/machine-function-properties.mir diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h index 965e8282ccab..3faffe5c4cab 100644 --- a/llvm/include/llvm/CodeGen/MachinePassManager.h +++ b/llvm/include/llvm/CodeGen/MachinePassManager.h @@ -16,6 +16,8 @@ // their respective analysis managers such as ModuleAnalysisManager and // FunctionAnalysisManager. // +// TODO: Add MachineFunctionProperties support. +// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_MACHINEPASSMANAGER_H @@ -42,67 +44,23 @@ using MachineFunctionAnalysisManager = AnalysisManager; /// automatically mixes in \c PassInfoMixin. template struct MachinePassInfoMixin : public PassInfoMixin { -protected: - class PropertyChanger { - MachineFunction &MF; - - template - using has_get_required_properties_t = - decltype(std::declval().getRequiredProperties()); - - template - using has_get_set_properties_t = - decltype(std::declval().getSetProperties()); - - template - using has_get_cleared_properties_t = - decltype(std::declval().getClearedProperties()); - - public: - PropertyChanger(MachineFunction &MF) : MF(MF) { -#ifndef NDEBUG - if constexpr (is_detected::value) { - auto &MFProps = MF.getProperties(); - auto RequiredProperties = DerivedT::getRequiredProperties(); - if (!MFProps.verifyRequiredProperties(RequiredProperties)) { - errs() << "MachineFunctionProperties required by " << DerivedT::name() - << " pass are not met by function " << MF.getName() << ".\n" - << "Required properties: "; - RequiredProperties.print(errs()); - errs() << "\nCurrent properties: "; - MFProps.print(errs()); - errs() << '\n'; - report_fatal_error("MachineFunctionProperties check failed"); - } -#endif - } - } - - ~PropertyChanger() { - if constexpr (is_detected::value) - MF.getProperties().set(DerivedT::getSetProperties()); - if constexpr (is_detected::value) - MF.getProperties().reset(DerivedT::getClearedProperties()); - } - }; - -public: - PreservedAnalyses runImpl(MachineFunction &MF, - MachineFunctionAnalysisManager &MFAM) { - PropertyChanger PC(MF); - return static_cast(this)->run(MF, MFAM); - } + // TODO: Add MachineFunctionProperties support. }; namespace detail { +struct MachinePassConcept + : PassConcept { + virtual MachineFunctionProperties getRequiredProperties() const = 0; + virtual MachineFunctionProperties getSetProperties() const = 0; + virtual MachineFunctionProperties getClearedProperties() const = 0; +}; -template -struct MachinePassModel - : PassModel { - explicit MachinePassModel(PassT &&Pass) - : PassModel( - std::move(Pass)) {} +template struct MachinePassModel : MachinePassConcept { + explicit MachinePassModel(PassT &&Pass) : Pass(std::move(Pass)) {} + // We have to explicitly define all the special member functions because MSVC + // refuses to generate them. + MachinePassModel(const MachinePassModel &Arg) : Pass(Arg.Pass) {} + MachinePassModel(MachinePassModel &&Arg) : Pass(std::move(Arg.Pass)) {} friend void swap(MachinePassModel &LHS, MachinePassModel &RHS) { using std::swap; @@ -117,8 +75,89 @@ struct MachinePassModel MachinePassModel &operator=(const MachinePassModel &) = delete; PreservedAnalyses run(MachineFunction &IR, MachineFunctionAnalysisManager &AM) override { - return this->Pass.runImpl(IR, AM); + return Pass.run(IR, AM); + } + + void printPipeline( + raw_ostream &OS, + function_ref MapClassName2PassName) override { + Pass.printPipeline(OS, MapClassName2PassName); + } + + StringRef name() const override { return PassT::name(); } + + template + using has_required_t = decltype(std::declval().isRequired()); + template + static std::enable_if_t::value, bool> + passIsRequiredImpl() { + return T::isRequired(); } + template + static std::enable_if_t::value, bool> + passIsRequiredImpl() { + return false; + } + bool isRequired() const override { return passIsRequiredImpl(); } + + template + using has_get_required_properties_t = + decltype(std::declval().getRequiredProperties()); + template + static std::enable_if_t::value, + MachineFunctionProperties> + getRequiredPropertiesImpl() { + return PassT::getRequiredProperties(); + } + template + static std::enable_if_t::value, + MachineFunctionProperties> + getRequiredPropertiesImpl() { + return MachineFunctionProperties(); + } + MachineFunctionProperties getRequiredProperties() const override { + return getRequiredPropertiesImpl(); + } + + template + using has_get_set_properties_t = + decltype(std::declval().getSetProperties()); + template + static std::enable_if_t::value, + MachineFunctionProperties> + getSetPropertiesImpl() { + return PassT::getSetProperties(); + } + template + static std::enable_if_t::value, + MachineFunctionProperties> + getSetPropertiesImpl() { + return MachineFunctionProperties(); + } + MachineFunctionProperties getSetProperties() const override { + return getSetPropertiesImpl(); + } + + template + using has_get_cleared_properties_t = + decltype(std::declval().getClearedProperties()); + template + static std::enable_if_t::value, + MachineFunctionProperties> + getClearedPropertiesImpl() { + return PassT::getClearedProperties(); + } + template + static std::enable_if_t::value, + MachineFunctionProperties> + getClearedPropertiesImpl() { + return MachineFunctionProperties(); + } + MachineFunctionProperties getClearedProperties() const override { + return getClearedPropertiesImpl(); + } + + PassT Pass; }; } // namespace detail @@ -212,12 +251,11 @@ private: class ModuleToMachineFunctionPassAdaptor : public PassInfoMixin { -public: - using PassConceptT = - detail::PassConcept; + using MachinePassConcept = detail::MachinePassConcept; +public: explicit ModuleToMachineFunctionPassAdaptor( - std::unique_ptr Pass) + std::unique_ptr Pass) : Pass(std::move(Pass)) {} /// Runs the function pass across every function in the module. @@ -228,39 +266,20 @@ public: static bool isRequired() { return true; } private: - std::unique_ptr Pass; + std::unique_ptr Pass; }; template ModuleToMachineFunctionPassAdaptor createModuleToMachineFunctionPassAdaptor(MachineFunctionPassT &&Pass) { - using PassModelT = detail::PassModel; + using PassModelT = detail::MachinePassModel; // Do not use make_unique, it causes too many template instantiations, // causing terrible compile times. return ModuleToMachineFunctionPassAdaptor( - std::unique_ptr( + std::unique_ptr( new PassModelT(std::forward(Pass)))); } -template <> -template -std::enable_if_t>::value> -PassManager::addPass(PassT &&Pass) { - using PassModelT = - detail::PassModel; - using MachinePassModelT = detail::MachinePassModel; - // Do not use make_unique or emplace_back, they cause too many template - // instantiations, causing terrible compile times. - if constexpr (std::is_base_of_v, PassT>) { - Passes.push_back(std::unique_ptr( - new MachinePassModelT(std::forward(Pass)))); - } else { - Passes.push_back(std::unique_ptr( - new PassModelT(std::forward(Pass)))); - } -} - template <> PreservedAnalyses PassManager::run(MachineFunction &, diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index 2f77ae655d9b..016602730e0e 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -127,8 +127,6 @@ MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass()) // MACHINE_FUNCTION_PASS("free-machine-function", FreeMachineFunctionPass()) MACHINE_FUNCTION_PASS("no-op-machine-function", NoOpMachineFunctionPass()) MACHINE_FUNCTION_PASS("print", PrintMIRPass()) -MACHINE_FUNCTION_PASS("require-all-machine-function-properties", - RequireAllMachineFunctionPropertiesPass()) #undef MACHINE_FUNCTION_PASS // After a pass is converted to new pass manager, its entry should be moved from diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 57975e34d426..f60f4eb3f0ef 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -365,33 +365,6 @@ public: static StringRef name() { return "TriggerVerifierErrorPass"; } }; -// A pass requires all MachineFunctionProperties. -// DO NOT USE THIS EXCEPT FOR TESTING! -class RequireAllMachineFunctionPropertiesPass - : public MachinePassInfoMixin { -public: - PreservedAnalyses run(MachineFunction &, MachineFunctionAnalysisManager &) { - return PreservedAnalyses::none(); - } - - static MachineFunctionProperties getRequiredProperties() { - MachineFunctionProperties MFProps; - MFProps.set(MachineFunctionProperties::Property::FailedISel); - MFProps.set(MachineFunctionProperties::Property::FailsVerification); - MFProps.set(MachineFunctionProperties::Property::IsSSA); - MFProps.set(MachineFunctionProperties::Property::Legalized); - MFProps.set(MachineFunctionProperties::Property::NoPHIs); - MFProps.set(MachineFunctionProperties::Property::NoVRegs); - MFProps.set(MachineFunctionProperties::Property::RegBankSelected); - MFProps.set(MachineFunctionProperties::Property::Selected); - MFProps.set(MachineFunctionProperties::Property::TiedOpsRewritten); - MFProps.set(MachineFunctionProperties::Property::TracksDebugUserValues); - MFProps.set(MachineFunctionProperties::Property::TracksLiveness); - return MFProps; - } - static StringRef name() { return "RequireAllMachineFunctionPropertiesPass"; } -}; - } // namespace PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO, diff --git a/llvm/test/tools/llc/new-pm/machine-function-properties.mir b/llvm/test/tools/llc/new-pm/machine-function-properties.mir deleted file mode 100644 index a9eb88ec6988..000000000000 --- a/llvm/test/tools/llc/new-pm/machine-function-properties.mir +++ /dev/null @@ -1,12 +0,0 @@ -# REQUIRES: asserts -# RUN: not --crash llc -mtriple=x86_64-pc-linux-gnu -passes=require-all-machine-function-properties -filetype=null %s 2>&1 | FileCheck %s - -# CHECK: MachineFunctionProperties required by RequireAllMachineFunctionPropertiesPass pass are not met by function f. - ---- -name: f -selected: false -body: | - bb.0: - RET 0 -... -- GitLab From 4f7132952222cd9b2709e98109d6ce0ef333940c Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 30 Mar 2024 10:12:14 +0800 Subject: [PATCH 0042/1690] Reland "[PassManager] Support MachineFunctionProperties (#83668)" (#87137) Fix preprocessor directive. --- .../include/llvm/CodeGen/MachinePassManager.h | 187 ++++++++---------- .../llvm/Passes/MachinePassRegistry.def | 2 + llvm/lib/Passes/PassBuilder.cpp | 27 +++ .../new-pm/machine-function-properties.mir | 12 ++ 4 files changed, 125 insertions(+), 103 deletions(-) create mode 100644 llvm/test/tools/llc/new-pm/machine-function-properties.mir diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h index 3faffe5c4cab..085a78f91917 100644 --- a/llvm/include/llvm/CodeGen/MachinePassManager.h +++ b/llvm/include/llvm/CodeGen/MachinePassManager.h @@ -16,8 +16,6 @@ // their respective analysis managers such as ModuleAnalysisManager and // FunctionAnalysisManager. // -// TODO: Add MachineFunctionProperties support. -// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_MACHINEPASSMANAGER_H @@ -44,23 +42,67 @@ using MachineFunctionAnalysisManager = AnalysisManager; /// automatically mixes in \c PassInfoMixin. template struct MachinePassInfoMixin : public PassInfoMixin { - // TODO: Add MachineFunctionProperties support. +protected: + class PropertyChanger { + MachineFunction &MF; + + template + using has_get_required_properties_t = + decltype(std::declval().getRequiredProperties()); + + template + using has_get_set_properties_t = + decltype(std::declval().getSetProperties()); + + template + using has_get_cleared_properties_t = + decltype(std::declval().getClearedProperties()); + + public: + PropertyChanger(MachineFunction &MF) : MF(MF) { +#ifndef NDEBUG + if constexpr (is_detected::value) { + auto &MFProps = MF.getProperties(); + auto RequiredProperties = DerivedT::getRequiredProperties(); + if (!MFProps.verifyRequiredProperties(RequiredProperties)) { + errs() << "MachineFunctionProperties required by " << DerivedT::name() + << " pass are not met by function " << MF.getName() << ".\n" + << "Required properties: "; + RequiredProperties.print(errs()); + errs() << "\nCurrent properties: "; + MFProps.print(errs()); + errs() << '\n'; + report_fatal_error("MachineFunctionProperties check failed"); + } + } +#endif + } + + ~PropertyChanger() { + if constexpr (is_detected::value) + MF.getProperties().set(DerivedT::getSetProperties()); + if constexpr (is_detected::value) + MF.getProperties().reset(DerivedT::getClearedProperties()); + } + }; + +public: + PreservedAnalyses runImpl(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM) { + PropertyChanger PC(MF); + return static_cast(this)->run(MF, MFAM); + } }; namespace detail { -struct MachinePassConcept - : PassConcept { - virtual MachineFunctionProperties getRequiredProperties() const = 0; - virtual MachineFunctionProperties getSetProperties() const = 0; - virtual MachineFunctionProperties getClearedProperties() const = 0; -}; -template struct MachinePassModel : MachinePassConcept { - explicit MachinePassModel(PassT &&Pass) : Pass(std::move(Pass)) {} - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - MachinePassModel(const MachinePassModel &Arg) : Pass(Arg.Pass) {} - MachinePassModel(MachinePassModel &&Arg) : Pass(std::move(Arg.Pass)) {} +template +struct MachinePassModel + : PassModel { + explicit MachinePassModel(PassT &&Pass) + : PassModel( + std::move(Pass)) {} friend void swap(MachinePassModel &LHS, MachinePassModel &RHS) { using std::swap; @@ -75,89 +117,8 @@ template struct MachinePassModel : MachinePassConcept { MachinePassModel &operator=(const MachinePassModel &) = delete; PreservedAnalyses run(MachineFunction &IR, MachineFunctionAnalysisManager &AM) override { - return Pass.run(IR, AM); - } - - void printPipeline( - raw_ostream &OS, - function_ref MapClassName2PassName) override { - Pass.printPipeline(OS, MapClassName2PassName); - } - - StringRef name() const override { return PassT::name(); } - - template - using has_required_t = decltype(std::declval().isRequired()); - template - static std::enable_if_t::value, bool> - passIsRequiredImpl() { - return T::isRequired(); + return this->Pass.runImpl(IR, AM); } - template - static std::enable_if_t::value, bool> - passIsRequiredImpl() { - return false; - } - bool isRequired() const override { return passIsRequiredImpl(); } - - template - using has_get_required_properties_t = - decltype(std::declval().getRequiredProperties()); - template - static std::enable_if_t::value, - MachineFunctionProperties> - getRequiredPropertiesImpl() { - return PassT::getRequiredProperties(); - } - template - static std::enable_if_t::value, - MachineFunctionProperties> - getRequiredPropertiesImpl() { - return MachineFunctionProperties(); - } - MachineFunctionProperties getRequiredProperties() const override { - return getRequiredPropertiesImpl(); - } - - template - using has_get_set_properties_t = - decltype(std::declval().getSetProperties()); - template - static std::enable_if_t::value, - MachineFunctionProperties> - getSetPropertiesImpl() { - return PassT::getSetProperties(); - } - template - static std::enable_if_t::value, - MachineFunctionProperties> - getSetPropertiesImpl() { - return MachineFunctionProperties(); - } - MachineFunctionProperties getSetProperties() const override { - return getSetPropertiesImpl(); - } - - template - using has_get_cleared_properties_t = - decltype(std::declval().getClearedProperties()); - template - static std::enable_if_t::value, - MachineFunctionProperties> - getClearedPropertiesImpl() { - return PassT::getClearedProperties(); - } - template - static std::enable_if_t::value, - MachineFunctionProperties> - getClearedPropertiesImpl() { - return MachineFunctionProperties(); - } - MachineFunctionProperties getClearedProperties() const override { - return getClearedPropertiesImpl(); - } - - PassT Pass; }; } // namespace detail @@ -251,11 +212,12 @@ private: class ModuleToMachineFunctionPassAdaptor : public PassInfoMixin { - using MachinePassConcept = detail::MachinePassConcept; - public: + using PassConceptT = + detail::PassConcept; + explicit ModuleToMachineFunctionPassAdaptor( - std::unique_ptr Pass) + std::unique_ptr Pass) : Pass(std::move(Pass)) {} /// Runs the function pass across every function in the module. @@ -266,20 +228,39 @@ public: static bool isRequired() { return true; } private: - std::unique_ptr Pass; + std::unique_ptr Pass; }; template ModuleToMachineFunctionPassAdaptor createModuleToMachineFunctionPassAdaptor(MachineFunctionPassT &&Pass) { - using PassModelT = detail::MachinePassModel; + using PassModelT = detail::PassModel; // Do not use make_unique, it causes too many template instantiations, // causing terrible compile times. return ModuleToMachineFunctionPassAdaptor( - std::unique_ptr( + std::unique_ptr( new PassModelT(std::forward(Pass)))); } +template <> +template +std::enable_if_t>::value> +PassManager::addPass(PassT &&Pass) { + using PassModelT = + detail::PassModel; + using MachinePassModelT = detail::MachinePassModel; + // Do not use make_unique or emplace_back, they cause too many template + // instantiations, causing terrible compile times. + if constexpr (std::is_base_of_v, PassT>) { + Passes.push_back(std::unique_ptr( + new MachinePassModelT(std::forward(Pass)))); + } else { + Passes.push_back(std::unique_ptr( + new PassModelT(std::forward(Pass)))); + } +} + template <> PreservedAnalyses PassManager::run(MachineFunction &, diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index 016602730e0e..2f77ae655d9b 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -127,6 +127,8 @@ MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass()) // MACHINE_FUNCTION_PASS("free-machine-function", FreeMachineFunctionPass()) MACHINE_FUNCTION_PASS("no-op-machine-function", NoOpMachineFunctionPass()) MACHINE_FUNCTION_PASS("print", PrintMIRPass()) +MACHINE_FUNCTION_PASS("require-all-machine-function-properties", + RequireAllMachineFunctionPropertiesPass()) #undef MACHINE_FUNCTION_PASS // After a pass is converted to new pass manager, its entry should be moved from diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index f60f4eb3f0ef..57975e34d426 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -365,6 +365,33 @@ public: static StringRef name() { return "TriggerVerifierErrorPass"; } }; +// A pass requires all MachineFunctionProperties. +// DO NOT USE THIS EXCEPT FOR TESTING! +class RequireAllMachineFunctionPropertiesPass + : public MachinePassInfoMixin { +public: + PreservedAnalyses run(MachineFunction &, MachineFunctionAnalysisManager &) { + return PreservedAnalyses::none(); + } + + static MachineFunctionProperties getRequiredProperties() { + MachineFunctionProperties MFProps; + MFProps.set(MachineFunctionProperties::Property::FailedISel); + MFProps.set(MachineFunctionProperties::Property::FailsVerification); + MFProps.set(MachineFunctionProperties::Property::IsSSA); + MFProps.set(MachineFunctionProperties::Property::Legalized); + MFProps.set(MachineFunctionProperties::Property::NoPHIs); + MFProps.set(MachineFunctionProperties::Property::NoVRegs); + MFProps.set(MachineFunctionProperties::Property::RegBankSelected); + MFProps.set(MachineFunctionProperties::Property::Selected); + MFProps.set(MachineFunctionProperties::Property::TiedOpsRewritten); + MFProps.set(MachineFunctionProperties::Property::TracksDebugUserValues); + MFProps.set(MachineFunctionProperties::Property::TracksLiveness); + return MFProps; + } + static StringRef name() { return "RequireAllMachineFunctionPropertiesPass"; } +}; + } // namespace PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO, diff --git a/llvm/test/tools/llc/new-pm/machine-function-properties.mir b/llvm/test/tools/llc/new-pm/machine-function-properties.mir new file mode 100644 index 000000000000..a9eb88ec6988 --- /dev/null +++ b/llvm/test/tools/llc/new-pm/machine-function-properties.mir @@ -0,0 +1,12 @@ +# REQUIRES: asserts +# RUN: not --crash llc -mtriple=x86_64-pc-linux-gnu -passes=require-all-machine-function-properties -filetype=null %s 2>&1 | FileCheck %s + +# CHECK: MachineFunctionProperties required by RequireAllMachineFunctionPropertiesPass pass are not met by function f. + +--- +name: f +selected: false +body: | + bb.0: + RET 0 +... -- GitLab From e9e4ab8147f0b8a8d1c8f1a70bef3999ffeeb651 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 30 Mar 2024 10:20:59 +0800 Subject: [PATCH 0043/1690] Revert "Reland "[PassManager] Support MachineFunctionProperties (#83668)"" (#87138) Reverts llvm/llvm-project#87137 It introduces ambiguous template specialization problem. Revert it. --- .../include/llvm/CodeGen/MachinePassManager.h | 187 ++++++++++-------- .../llvm/Passes/MachinePassRegistry.def | 2 - llvm/lib/Passes/PassBuilder.cpp | 27 --- .../new-pm/machine-function-properties.mir | 12 -- 4 files changed, 103 insertions(+), 125 deletions(-) delete mode 100644 llvm/test/tools/llc/new-pm/machine-function-properties.mir diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h index 085a78f91917..3faffe5c4cab 100644 --- a/llvm/include/llvm/CodeGen/MachinePassManager.h +++ b/llvm/include/llvm/CodeGen/MachinePassManager.h @@ -16,6 +16,8 @@ // their respective analysis managers such as ModuleAnalysisManager and // FunctionAnalysisManager. // +// TODO: Add MachineFunctionProperties support. +// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_MACHINEPASSMANAGER_H @@ -42,67 +44,23 @@ using MachineFunctionAnalysisManager = AnalysisManager; /// automatically mixes in \c PassInfoMixin. template struct MachinePassInfoMixin : public PassInfoMixin { -protected: - class PropertyChanger { - MachineFunction &MF; - - template - using has_get_required_properties_t = - decltype(std::declval().getRequiredProperties()); - - template - using has_get_set_properties_t = - decltype(std::declval().getSetProperties()); - - template - using has_get_cleared_properties_t = - decltype(std::declval().getClearedProperties()); - - public: - PropertyChanger(MachineFunction &MF) : MF(MF) { -#ifndef NDEBUG - if constexpr (is_detected::value) { - auto &MFProps = MF.getProperties(); - auto RequiredProperties = DerivedT::getRequiredProperties(); - if (!MFProps.verifyRequiredProperties(RequiredProperties)) { - errs() << "MachineFunctionProperties required by " << DerivedT::name() - << " pass are not met by function " << MF.getName() << ".\n" - << "Required properties: "; - RequiredProperties.print(errs()); - errs() << "\nCurrent properties: "; - MFProps.print(errs()); - errs() << '\n'; - report_fatal_error("MachineFunctionProperties check failed"); - } - } -#endif - } - - ~PropertyChanger() { - if constexpr (is_detected::value) - MF.getProperties().set(DerivedT::getSetProperties()); - if constexpr (is_detected::value) - MF.getProperties().reset(DerivedT::getClearedProperties()); - } - }; - -public: - PreservedAnalyses runImpl(MachineFunction &MF, - MachineFunctionAnalysisManager &MFAM) { - PropertyChanger PC(MF); - return static_cast(this)->run(MF, MFAM); - } + // TODO: Add MachineFunctionProperties support. }; namespace detail { +struct MachinePassConcept + : PassConcept { + virtual MachineFunctionProperties getRequiredProperties() const = 0; + virtual MachineFunctionProperties getSetProperties() const = 0; + virtual MachineFunctionProperties getClearedProperties() const = 0; +}; -template -struct MachinePassModel - : PassModel { - explicit MachinePassModel(PassT &&Pass) - : PassModel( - std::move(Pass)) {} +template struct MachinePassModel : MachinePassConcept { + explicit MachinePassModel(PassT &&Pass) : Pass(std::move(Pass)) {} + // We have to explicitly define all the special member functions because MSVC + // refuses to generate them. + MachinePassModel(const MachinePassModel &Arg) : Pass(Arg.Pass) {} + MachinePassModel(MachinePassModel &&Arg) : Pass(std::move(Arg.Pass)) {} friend void swap(MachinePassModel &LHS, MachinePassModel &RHS) { using std::swap; @@ -117,8 +75,89 @@ struct MachinePassModel MachinePassModel &operator=(const MachinePassModel &) = delete; PreservedAnalyses run(MachineFunction &IR, MachineFunctionAnalysisManager &AM) override { - return this->Pass.runImpl(IR, AM); + return Pass.run(IR, AM); + } + + void printPipeline( + raw_ostream &OS, + function_ref MapClassName2PassName) override { + Pass.printPipeline(OS, MapClassName2PassName); + } + + StringRef name() const override { return PassT::name(); } + + template + using has_required_t = decltype(std::declval().isRequired()); + template + static std::enable_if_t::value, bool> + passIsRequiredImpl() { + return T::isRequired(); } + template + static std::enable_if_t::value, bool> + passIsRequiredImpl() { + return false; + } + bool isRequired() const override { return passIsRequiredImpl(); } + + template + using has_get_required_properties_t = + decltype(std::declval().getRequiredProperties()); + template + static std::enable_if_t::value, + MachineFunctionProperties> + getRequiredPropertiesImpl() { + return PassT::getRequiredProperties(); + } + template + static std::enable_if_t::value, + MachineFunctionProperties> + getRequiredPropertiesImpl() { + return MachineFunctionProperties(); + } + MachineFunctionProperties getRequiredProperties() const override { + return getRequiredPropertiesImpl(); + } + + template + using has_get_set_properties_t = + decltype(std::declval().getSetProperties()); + template + static std::enable_if_t::value, + MachineFunctionProperties> + getSetPropertiesImpl() { + return PassT::getSetProperties(); + } + template + static std::enable_if_t::value, + MachineFunctionProperties> + getSetPropertiesImpl() { + return MachineFunctionProperties(); + } + MachineFunctionProperties getSetProperties() const override { + return getSetPropertiesImpl(); + } + + template + using has_get_cleared_properties_t = + decltype(std::declval().getClearedProperties()); + template + static std::enable_if_t::value, + MachineFunctionProperties> + getClearedPropertiesImpl() { + return PassT::getClearedProperties(); + } + template + static std::enable_if_t::value, + MachineFunctionProperties> + getClearedPropertiesImpl() { + return MachineFunctionProperties(); + } + MachineFunctionProperties getClearedProperties() const override { + return getClearedPropertiesImpl(); + } + + PassT Pass; }; } // namespace detail @@ -212,12 +251,11 @@ private: class ModuleToMachineFunctionPassAdaptor : public PassInfoMixin { -public: - using PassConceptT = - detail::PassConcept; + using MachinePassConcept = detail::MachinePassConcept; +public: explicit ModuleToMachineFunctionPassAdaptor( - std::unique_ptr Pass) + std::unique_ptr Pass) : Pass(std::move(Pass)) {} /// Runs the function pass across every function in the module. @@ -228,39 +266,20 @@ public: static bool isRequired() { return true; } private: - std::unique_ptr Pass; + std::unique_ptr Pass; }; template ModuleToMachineFunctionPassAdaptor createModuleToMachineFunctionPassAdaptor(MachineFunctionPassT &&Pass) { - using PassModelT = detail::PassModel; + using PassModelT = detail::MachinePassModel; // Do not use make_unique, it causes too many template instantiations, // causing terrible compile times. return ModuleToMachineFunctionPassAdaptor( - std::unique_ptr( + std::unique_ptr( new PassModelT(std::forward(Pass)))); } -template <> -template -std::enable_if_t>::value> -PassManager::addPass(PassT &&Pass) { - using PassModelT = - detail::PassModel; - using MachinePassModelT = detail::MachinePassModel; - // Do not use make_unique or emplace_back, they cause too many template - // instantiations, causing terrible compile times. - if constexpr (std::is_base_of_v, PassT>) { - Passes.push_back(std::unique_ptr( - new MachinePassModelT(std::forward(Pass)))); - } else { - Passes.push_back(std::unique_ptr( - new PassModelT(std::forward(Pass)))); - } -} - template <> PreservedAnalyses PassManager::run(MachineFunction &, diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index 2f77ae655d9b..016602730e0e 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -127,8 +127,6 @@ MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass()) // MACHINE_FUNCTION_PASS("free-machine-function", FreeMachineFunctionPass()) MACHINE_FUNCTION_PASS("no-op-machine-function", NoOpMachineFunctionPass()) MACHINE_FUNCTION_PASS("print", PrintMIRPass()) -MACHINE_FUNCTION_PASS("require-all-machine-function-properties", - RequireAllMachineFunctionPropertiesPass()) #undef MACHINE_FUNCTION_PASS // After a pass is converted to new pass manager, its entry should be moved from diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 57975e34d426..f60f4eb3f0ef 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -365,33 +365,6 @@ public: static StringRef name() { return "TriggerVerifierErrorPass"; } }; -// A pass requires all MachineFunctionProperties. -// DO NOT USE THIS EXCEPT FOR TESTING! -class RequireAllMachineFunctionPropertiesPass - : public MachinePassInfoMixin { -public: - PreservedAnalyses run(MachineFunction &, MachineFunctionAnalysisManager &) { - return PreservedAnalyses::none(); - } - - static MachineFunctionProperties getRequiredProperties() { - MachineFunctionProperties MFProps; - MFProps.set(MachineFunctionProperties::Property::FailedISel); - MFProps.set(MachineFunctionProperties::Property::FailsVerification); - MFProps.set(MachineFunctionProperties::Property::IsSSA); - MFProps.set(MachineFunctionProperties::Property::Legalized); - MFProps.set(MachineFunctionProperties::Property::NoPHIs); - MFProps.set(MachineFunctionProperties::Property::NoVRegs); - MFProps.set(MachineFunctionProperties::Property::RegBankSelected); - MFProps.set(MachineFunctionProperties::Property::Selected); - MFProps.set(MachineFunctionProperties::Property::TiedOpsRewritten); - MFProps.set(MachineFunctionProperties::Property::TracksDebugUserValues); - MFProps.set(MachineFunctionProperties::Property::TracksLiveness); - return MFProps; - } - static StringRef name() { return "RequireAllMachineFunctionPropertiesPass"; } -}; - } // namespace PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO, diff --git a/llvm/test/tools/llc/new-pm/machine-function-properties.mir b/llvm/test/tools/llc/new-pm/machine-function-properties.mir deleted file mode 100644 index a9eb88ec6988..000000000000 --- a/llvm/test/tools/llc/new-pm/machine-function-properties.mir +++ /dev/null @@ -1,12 +0,0 @@ -# REQUIRES: asserts -# RUN: not --crash llc -mtriple=x86_64-pc-linux-gnu -passes=require-all-machine-function-properties -filetype=null %s 2>&1 | FileCheck %s - -# CHECK: MachineFunctionProperties required by RequireAllMachineFunctionPropertiesPass pass are not met by function f. - ---- -name: f -selected: false -body: | - bb.0: - RET 0 -... -- GitLab From 09403a66c01ef189c5e6993410fc9e082acaccf2 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Fri, 29 Mar 2024 23:26:29 -0700 Subject: [PATCH 0044/1690] [ELF,test] Improve "non-exported symbol" tests Test that the error is not triggered when we see the non-exported definition first (#86777). --- lld/test/ELF/allow-shlib-undefined.s | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lld/test/ELF/allow-shlib-undefined.s b/lld/test/ELF/allow-shlib-undefined.s index 4b7151c8bc0d..053cb80e2b9d 100644 --- a/lld/test/ELF/allow-shlib-undefined.s +++ b/lld/test/ELF/allow-shlib-undefined.s @@ -31,10 +31,14 @@ ## Test some cases when a relocatable object file provides a non-exported definition. # RUN: not ld.lld main.o a.so def-hidden.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=NONEXPORTED +## TODO +# RUN: ld.lld main.o def-hidden.o a.so -o /dev/null # RUN: not ld.lld main.o a.so def-hidden.o -shared --no-allow-shlib-undefined -o /dev/null 2>&1 | FileCheck %s --check-prefix=NONEXPORTED # RUN: ld.lld main.o a.so def-hidden.o --allow-shlib-undefined --fatal-warnings -o /dev/null ## Test a relocatable object file definition that is converted to STB_LOCAL. # RUN: not ld.lld main.o a.so def-hidden.o --version-script=local.ver -o /dev/null 2>&1 | FileCheck %s --check-prefix=NONEXPORTED +## TODO +# RUN: ld.lld main.o def-hidden.o a.so --version-script=local.ver -o /dev/null ## The section containing the definition is discarded, and we report an error. # RUN: not ld.lld --gc-sections main.o a.so def-hidden.o -o /dev/null 2>&1 | FileCheck %s -- GitLab From df54f627fad789ccb11c72a9fddf116decbeba0e Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Fri, 29 Mar 2024 23:36:50 -0700 Subject: [PATCH 0045/1690] [ELF] Enhance --no-allow-shlib-undefined for non-exported definitions For a DSO with all DT_NEEDED entries accounted for, if it contains an undefined non-weak symbol that shares a name with a non-exported definition (hidden visibility or localized by a version script), and there is no DSO definition, we should report an error. #70769 implemented the error when we see `ref.so def-hidden.so`. This patch implementes the error when we see `def-hidden.so ref.so`, matching GNU ld. Close #86777 --- lld/ELF/InputFiles.cpp | 2 +- lld/test/ELF/allow-shlib-undefined.s | 6 ++---- lld/test/ELF/shlib-undefined-local.s | 5 ++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index 725c6f166fff..6529ea072fae 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -1557,7 +1557,7 @@ template void SharedFile::parse() { Symbol *s = symtab.addSymbol( Undefined{this, name, sym.getBinding(), sym.st_other, sym.getType()}); s->exportDynamic = true; - if (s->isUndefined() && sym.getBinding() != STB_WEAK && + if (sym.getBinding() != STB_WEAK && config->unresolvedSymbolsInShlib != UnresolvedPolicy::Ignore) requiredSymbols.push_back(s); continue; diff --git a/lld/test/ELF/allow-shlib-undefined.s b/lld/test/ELF/allow-shlib-undefined.s index 053cb80e2b9d..c69c1ea20ce3 100644 --- a/lld/test/ELF/allow-shlib-undefined.s +++ b/lld/test/ELF/allow-shlib-undefined.s @@ -31,14 +31,12 @@ ## Test some cases when a relocatable object file provides a non-exported definition. # RUN: not ld.lld main.o a.so def-hidden.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=NONEXPORTED -## TODO -# RUN: ld.lld main.o def-hidden.o a.so -o /dev/null +# RUN: not ld.lld main.o def-hidden.o a.so -o /dev/null 2>&1 | FileCheck %s --check-prefix=NONEXPORTED # RUN: not ld.lld main.o a.so def-hidden.o -shared --no-allow-shlib-undefined -o /dev/null 2>&1 | FileCheck %s --check-prefix=NONEXPORTED # RUN: ld.lld main.o a.so def-hidden.o --allow-shlib-undefined --fatal-warnings -o /dev/null ## Test a relocatable object file definition that is converted to STB_LOCAL. # RUN: not ld.lld main.o a.so def-hidden.o --version-script=local.ver -o /dev/null 2>&1 | FileCheck %s --check-prefix=NONEXPORTED -## TODO -# RUN: ld.lld main.o def-hidden.o a.so --version-script=local.ver -o /dev/null +# RUN: not ld.lld main.o def-hidden.o a.so --version-script=local.ver -o /dev/null 2>&1 | FileCheck %s --check-prefix=NONEXPORTED ## The section containing the definition is discarded, and we report an error. # RUN: not ld.lld --gc-sections main.o a.so def-hidden.o -o /dev/null 2>&1 | FileCheck %s diff --git a/lld/test/ELF/shlib-undefined-local.s b/lld/test/ELF/shlib-undefined-local.s index 8fceec1bf60f..6d3e8da34e29 100644 --- a/lld/test/ELF/shlib-undefined-local.s +++ b/lld/test/ELF/shlib-undefined-local.s @@ -5,10 +5,9 @@ # RUN: llvm-mc -filetype=obj -triple=x86_64-linux-gnu -o %t2.o %s # RUN: echo "{ local: *; };" > %t.script -# RUN: ld.lld -version-script %t.script -o %t %t2.o %t.so -# RUN: llvm-nm -g %t | FileCheck -allow-empty %s +# RUN: not ld.lld -version-script %t.script %t2.o %t.so -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR -# CHECK-NOT: should_not_be_exported +# ERR: error: non-exported symbol 'should_not_be_exported' in '{{.*}}tmp2.o' is referenced by DSO '{{.*}}tmp.so' .globl should_not_be_exported should_not_be_exported: -- GitLab From 95258419f6fe2e0922c2c0916fd176b9f7361555 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Sat, 30 Mar 2024 08:01:18 +0000 Subject: [PATCH 0046/1690] [AMDGPU] Use AMDGPU::isIntrinsicAlwaysUniform in isSDNodeAlwaysUniform (#87085) This is mostly just a simplification, but tests show a slight codegen improvement in code using the deprecated amdgcn.icmp/fcmp intrinsics. --- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 7 +--- llvm/test/CodeGen/AMDGPU/wave32.ll | 34 +++++++------------ 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index bee43b6c18c8..f283af6fa07d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -851,12 +851,7 @@ bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode *N) const { return true; case ISD::INTRINSIC_WO_CHAIN: { unsigned IntrID = N->getConstantOperandVal(0); - switch (IntrID) { - case Intrinsic::amdgcn_readfirstlane: - case Intrinsic::amdgcn_readlane: - return true; - } - return false; + return AMDGPU::isIntrinsicAlwaysUniform(IntrID); } case ISD::LOAD: if (cast(N)->getMemOperand()->getAddrSpace() == diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll index 82816b4564e8..901e88a4c6ac 100644 --- a/llvm/test/CodeGen/AMDGPU/wave32.ll +++ b/llvm/test/CodeGen/AMDGPU/wave32.ll @@ -2479,8 +2479,7 @@ define amdgpu_kernel void @icmp64(i32 %n, i32 %s) { ; GFX1032-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ; GFX1032-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX1032-NEXT: v_mul_lo_u32 v2, s1, v1 -; GFX1032-NEXT: s_ff1_i32_b32 s1, 0x80000000 -; GFX1032-NEXT: s_add_i32 s1, s1, 32 +; GFX1032-NEXT: s_brev_b32 s1, 1 ; GFX1032-NEXT: v_mul_hi_u32 v2, v1, v2 ; GFX1032-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX1032-NEXT: v_mul_hi_u32 v1, v0, v1 @@ -2494,8 +2493,7 @@ define amdgpu_kernel void @icmp64(i32 %n, i32 %s) { ; GFX1032-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo ; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 ; GFX1032-NEXT: s_lshr_b32 s0, vcc_lo, 1 -; GFX1032-NEXT: s_ff1_i32_b32 s0, s0 -; GFX1032-NEXT: s_min_u32 s0, s0, s1 +; GFX1032-NEXT: s_ff1_i32_b64 s0, s[0:1] ; GFX1032-NEXT: s_cmp_gt_u32 s0, 9 ; GFX1032-NEXT: s_cselect_b32 s0, -1, 0 ; GFX1032-NEXT: s_and_b32 s0, vcc_lo, s0 @@ -2529,10 +2527,7 @@ define amdgpu_kernel void @icmp64(i32 %n, i32 %s) { ; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX1064-NEXT: s_lshr_b64 s[0:1], vcc, 1 ; GFX1064-NEXT: s_bitset1_b32 s1, 31 -; GFX1064-NEXT: s_ff1_i32_b32 s0, s0 -; GFX1064-NEXT: s_ff1_i32_b32 s1, s1 -; GFX1064-NEXT: s_add_i32 s1, s1, 32 -; GFX1064-NEXT: s_min_u32 s0, s0, s1 +; GFX1064-NEXT: s_ff1_i32_b64 s0, s[0:1] ; GFX1064-NEXT: s_cmp_gt_u32 s0, 9 ; GFX1064-NEXT: s_cselect_b64 s[0:1], -1, 0 ; GFX1064-NEXT: s_and_b64 s[0:1], vcc, s[0:1] @@ -2576,9 +2571,8 @@ define amdgpu_kernel void @fcmp64(float %n, float %s) { ; GFX1032-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-NEXT: v_div_scale_f32 v1, s1, s0, s0, v0 ; GFX1032-NEXT: v_div_scale_f32 v4, vcc_lo, v0, s0, v0 -; GFX1032-NEXT: s_ff1_i32_b32 s1, 0x80000000 +; GFX1032-NEXT: s_brev_b32 s1, 1 ; GFX1032-NEXT: v_rcp_f32_e32 v2, v1 -; GFX1032-NEXT: s_add_i32 s1, s1, 32 ; GFX1032-NEXT: v_fma_f32 v3, -v1, v2, 1.0 ; GFX1032-NEXT: v_fmac_f32_e32 v2, v3, v2 ; GFX1032-NEXT: v_mul_f32_e32 v3, v4, v2 @@ -2592,8 +2586,7 @@ define amdgpu_kernel void @fcmp64(float %n, float %s) { ; GFX1032-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0 ; GFX1032-NEXT: s_lshr_b32 s0, vcc_lo, 1 ; GFX1032-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v0 -; GFX1032-NEXT: s_ff1_i32_b32 s0, s0 -; GFX1032-NEXT: s_min_u32 s0, s0, s1 +; GFX1032-NEXT: s_ff1_i32_b64 s0, s[0:1] ; GFX1032-NEXT: s_cmp_gt_u32 s0, 9 ; GFX1032-NEXT: s_cselect_b32 s0, -1, 0 ; GFX1032-NEXT: s_and_b32 s0, vcc_lo, s0 @@ -2609,15 +2602,15 @@ define amdgpu_kernel void @fcmp64(float %n, float %s) { ; GFX1064-NEXT: v_cvt_f32_u32_e32 v0, v0 ; GFX1064-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-NEXT: v_div_scale_f32 v1, s[0:1], s2, s2, v0 -; GFX1064-NEXT: v_div_scale_f32 v4, vcc, v0, s2, v0 ; GFX1064-NEXT: v_rcp_f32_e32 v2, v1 ; GFX1064-NEXT: v_fma_f32 v3, -v1, v2, 1.0 ; GFX1064-NEXT: v_fmac_f32_e32 v2, v3, v2 -; GFX1064-NEXT: v_mul_f32_e32 v3, v4, v2 -; GFX1064-NEXT: v_fma_f32 v5, -v1, v3, v4 -; GFX1064-NEXT: v_fmac_f32_e32 v3, v5, v2 -; GFX1064-NEXT: v_fma_f32 v1, -v1, v3, v4 -; GFX1064-NEXT: v_div_fmas_f32 v1, v1, v2, v3 +; GFX1064-NEXT: v_div_scale_f32 v3, vcc, v0, s2, v0 +; GFX1064-NEXT: v_mul_f32_e32 v4, v3, v2 +; GFX1064-NEXT: v_fma_f32 v5, -v1, v4, v3 +; GFX1064-NEXT: v_fmac_f32_e32 v4, v5, v2 +; GFX1064-NEXT: v_fma_f32 v1, -v1, v4, v3 +; GFX1064-NEXT: v_div_fmas_f32 v1, v1, v2, v4 ; GFX1064-NEXT: v_div_fixup_f32 v1, v1, s2, v0 ; GFX1064-NEXT: v_trunc_f32_e32 v1, v1 ; GFX1064-NEXT: v_fma_f32 v0, -v1, s2, v0 @@ -2625,10 +2618,7 @@ define amdgpu_kernel void @fcmp64(float %n, float %s) { ; GFX1064-NEXT: s_lshr_b64 s[0:1], vcc, 1 ; GFX1064-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0 ; GFX1064-NEXT: s_bitset1_b32 s1, 31 -; GFX1064-NEXT: s_ff1_i32_b32 s0, s0 -; GFX1064-NEXT: s_ff1_i32_b32 s1, s1 -; GFX1064-NEXT: s_add_i32 s1, s1, 32 -; GFX1064-NEXT: s_min_u32 s0, s0, s1 +; GFX1064-NEXT: s_ff1_i32_b64 s0, s[0:1] ; GFX1064-NEXT: s_cmp_gt_u32 s0, 9 ; GFX1064-NEXT: s_cselect_b64 s[0:1], -1, 0 ; GFX1064-NEXT: s_and_b64 s[0:1], vcc, s[0:1] -- GitLab From 631ae59d301496f86a8d46d28f7b12afbd12a3c1 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Sat, 30 Mar 2024 09:53:19 +0100 Subject: [PATCH 0047/1690] [MLIR][ExecutionEngine] Introduce shared library (#87067) This commit introduces a shared library for the MLIR execution engine. This library is only built when `LLVM_BUILD_LLVM_DYLIB` is set. Having such a library allows downstream users to depend on the execution engine without giving up dynamic linkage. This is especially important for CPU runner-style tools, as they link against large parts of MLIR and LLVM. It is alternatively possible to modify the `MLIRExecutionEngine` target when `LLVM_BUILD_LLVM_DYLIB` is set, to avoid duplicated libraries. --- mlir/lib/ExecutionEngine/CMakeLists.txt | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/mlir/lib/ExecutionEngine/CMakeLists.txt b/mlir/lib/ExecutionEngine/CMakeLists.txt index b7e448d5417e..a091944b9ee7 100644 --- a/mlir/lib/ExecutionEngine/CMakeLists.txt +++ b/mlir/lib/ExecutionEngine/CMakeLists.txt @@ -97,6 +97,29 @@ add_mlir_library(MLIRExecutionEngine MLIRTargetLLVMIRExport ) +if(LLVM_BUILD_LLVM_DYLIB) + # Build a shared library for the execution engine. Some downstream projects + # use this library to build their own CPU runners while preserving dynamic + # linkage. + add_mlir_library(MLIRExecutionEngineShared + ExecutionEngine.cpp + SHARED + + EXCLUDE_FROM_LIBMLIR + + ADDITIONAL_HEADER_DIRS + ${MLIR_MAIN_INCLUDE_DIR}/mlir/ExecutionEngine + + # Ensures that all necessary dependencies are resolved. + DEPENDS + MLIRExecutionEngine + + LINK_LIBS PUBLIC + LLVM + MLIR + ) +endif() + get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS) add_mlir_library(MLIRJitRunner JitRunner.cpp -- GitLab From 546dc2245ffc4cccd0b05b58b7a5955e355a3b27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Sat, 30 Mar 2024 10:29:56 +0100 Subject: [PATCH 0048/1690] [clang-repl] Minor cleanups in Value.cpp (NFC) (#87066) --- clang/lib/Interpreter/Value.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/clang/lib/Interpreter/Value.cpp b/clang/lib/Interpreter/Value.cpp index 1d6b2da087e9..eb2ce9c9fd33 100644 --- a/clang/lib/Interpreter/Value.cpp +++ b/clang/lib/Interpreter/Value.cpp @@ -1,4 +1,4 @@ -//===--- Interpreter.h - Incremental Compiation and Execution---*- C++ -*-===// +//===------------ Value.cpp - Definition of interpreter value -------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -22,8 +22,6 @@ #include #include -using namespace clang; - namespace { // This is internal buffer maintained by Value, used to hold temporaries. @@ -61,7 +59,7 @@ public: void Release() { assert(RefCnt > 0 && "Can't release if reference count is already zero"); if (--RefCnt == 0) { - // We hace a non-trivial dtor. + // We have a non-trivial dtor. if (Dtor && IsAlive()) { assert(Elements && "We at least should have 1 element in Value"); size_t Stride = AllocSize / Elements; @@ -97,6 +95,8 @@ private: }; } // namespace +namespace clang { + static Value::Kind ConvertQualTypeToKind(const ASTContext &Ctx, QualType QT) { if (Ctx.hasSameType(QT, Ctx.VoidTy)) return Value::K_Void; @@ -265,3 +265,5 @@ void Value::print(llvm::raw_ostream &Out) const { assert(OpaqueType != nullptr && "Can't print default Value"); Out << "Not implement yet.\n"; } + +} // namespace clang -- GitLab From b361b5369ed4bb186f2a11459b1bc1d54d73a33e Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 30 Mar 2024 20:03:10 +0800 Subject: [PATCH 0049/1690] Reland "[PassManager] Support MachineFunctionProperties (#83668)" (#87141) Unfortunately GCC 9 rejects code in https://godbolt.org/z/zd9r5GM3e GCC 11 accepts this code. --- .../include/llvm/CodeGen/MachinePassManager.h | 189 ++++++++---------- llvm/include/llvm/IR/PassManager.h | 37 ++-- .../llvm/Passes/MachinePassRegistry.def | 2 + llvm/lib/Passes/PassBuilder.cpp | 27 +++ .../new-pm/machine-function-properties.mir | 12 ++ 5 files changed, 143 insertions(+), 124 deletions(-) create mode 100644 llvm/test/tools/llc/new-pm/machine-function-properties.mir diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h index 3faffe5c4cab..8689fd19030f 100644 --- a/llvm/include/llvm/CodeGen/MachinePassManager.h +++ b/llvm/include/llvm/CodeGen/MachinePassManager.h @@ -16,8 +16,6 @@ // their respective analysis managers such as ModuleAnalysisManager and // FunctionAnalysisManager. // -// TODO: Add MachineFunctionProperties support. -// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_MACHINEPASSMANAGER_H @@ -44,23 +42,67 @@ using MachineFunctionAnalysisManager = AnalysisManager; /// automatically mixes in \c PassInfoMixin. template struct MachinePassInfoMixin : public PassInfoMixin { - // TODO: Add MachineFunctionProperties support. +protected: + class PropertyChanger { + MachineFunction &MF; + + template + using has_get_required_properties_t = + decltype(std::declval().getRequiredProperties()); + + template + using has_get_set_properties_t = + decltype(std::declval().getSetProperties()); + + template + using has_get_cleared_properties_t = + decltype(std::declval().getClearedProperties()); + + public: + PropertyChanger(MachineFunction &MF) : MF(MF) { +#ifndef NDEBUG + if constexpr (is_detected::value) { + auto &MFProps = MF.getProperties(); + auto RequiredProperties = DerivedT::getRequiredProperties(); + if (!MFProps.verifyRequiredProperties(RequiredProperties)) { + errs() << "MachineFunctionProperties required by " << DerivedT::name() + << " pass are not met by function " << MF.getName() << ".\n" + << "Required properties: "; + RequiredProperties.print(errs()); + errs() << "\nCurrent properties: "; + MFProps.print(errs()); + errs() << '\n'; + report_fatal_error("MachineFunctionProperties check failed"); + } + } +#endif + } + + ~PropertyChanger() { + if constexpr (is_detected::value) + MF.getProperties().set(DerivedT::getSetProperties()); + if constexpr (is_detected::value) + MF.getProperties().reset(DerivedT::getClearedProperties()); + } + }; + +public: + PreservedAnalyses runImpl(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM) { + PropertyChanger PC(MF); + return static_cast(this)->run(MF, MFAM); + } }; namespace detail { -struct MachinePassConcept - : PassConcept { - virtual MachineFunctionProperties getRequiredProperties() const = 0; - virtual MachineFunctionProperties getSetProperties() const = 0; - virtual MachineFunctionProperties getClearedProperties() const = 0; -}; -template struct MachinePassModel : MachinePassConcept { - explicit MachinePassModel(PassT &&Pass) : Pass(std::move(Pass)) {} - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - MachinePassModel(const MachinePassModel &Arg) : Pass(Arg.Pass) {} - MachinePassModel(MachinePassModel &&Arg) : Pass(std::move(Arg.Pass)) {} +template +struct MachinePassModel + : PassModel { + explicit MachinePassModel(PassT &&Pass) + : PassModel( + std::move(Pass)) {} friend void swap(MachinePassModel &LHS, MachinePassModel &RHS) { using std::swap; @@ -75,89 +117,8 @@ template struct MachinePassModel : MachinePassConcept { MachinePassModel &operator=(const MachinePassModel &) = delete; PreservedAnalyses run(MachineFunction &IR, MachineFunctionAnalysisManager &AM) override { - return Pass.run(IR, AM); - } - - void printPipeline( - raw_ostream &OS, - function_ref MapClassName2PassName) override { - Pass.printPipeline(OS, MapClassName2PassName); - } - - StringRef name() const override { return PassT::name(); } - - template - using has_required_t = decltype(std::declval().isRequired()); - template - static std::enable_if_t::value, bool> - passIsRequiredImpl() { - return T::isRequired(); + return this->Pass.runImpl(IR, AM); } - template - static std::enable_if_t::value, bool> - passIsRequiredImpl() { - return false; - } - bool isRequired() const override { return passIsRequiredImpl(); } - - template - using has_get_required_properties_t = - decltype(std::declval().getRequiredProperties()); - template - static std::enable_if_t::value, - MachineFunctionProperties> - getRequiredPropertiesImpl() { - return PassT::getRequiredProperties(); - } - template - static std::enable_if_t::value, - MachineFunctionProperties> - getRequiredPropertiesImpl() { - return MachineFunctionProperties(); - } - MachineFunctionProperties getRequiredProperties() const override { - return getRequiredPropertiesImpl(); - } - - template - using has_get_set_properties_t = - decltype(std::declval().getSetProperties()); - template - static std::enable_if_t::value, - MachineFunctionProperties> - getSetPropertiesImpl() { - return PassT::getSetProperties(); - } - template - static std::enable_if_t::value, - MachineFunctionProperties> - getSetPropertiesImpl() { - return MachineFunctionProperties(); - } - MachineFunctionProperties getSetProperties() const override { - return getSetPropertiesImpl(); - } - - template - using has_get_cleared_properties_t = - decltype(std::declval().getClearedProperties()); - template - static std::enable_if_t::value, - MachineFunctionProperties> - getClearedPropertiesImpl() { - return PassT::getClearedProperties(); - } - template - static std::enable_if_t::value, - MachineFunctionProperties> - getClearedPropertiesImpl() { - return MachineFunctionProperties(); - } - MachineFunctionProperties getClearedProperties() const override { - return getClearedPropertiesImpl(); - } - - PassT Pass; }; } // namespace detail @@ -251,11 +212,12 @@ private: class ModuleToMachineFunctionPassAdaptor : public PassInfoMixin { - using MachinePassConcept = detail::MachinePassConcept; - public: + using PassConceptT = + detail::PassConcept; + explicit ModuleToMachineFunctionPassAdaptor( - std::unique_ptr Pass) + std::unique_ptr Pass) : Pass(std::move(Pass)) {} /// Runs the function pass across every function in the module. @@ -266,20 +228,41 @@ public: static bool isRequired() { return true; } private: - std::unique_ptr Pass; + std::unique_ptr Pass; }; template ModuleToMachineFunctionPassAdaptor createModuleToMachineFunctionPassAdaptor(MachineFunctionPassT &&Pass) { - using PassModelT = detail::MachinePassModel; + using PassModelT = detail::PassModel; // Do not use make_unique, it causes too many template instantiations, // causing terrible compile times. return ModuleToMachineFunctionPassAdaptor( - std::unique_ptr( + std::unique_ptr( new PassModelT(std::forward(Pass)))); } +template <> +template +void PassManager::addPass(PassT &&Pass) { + using PassModelT = + detail::PassModel; + using MachinePassModelT = detail::MachinePassModel; + // Do not use make_unique or emplace_back, they cause too many template + // instantiations, causing terrible compile times. + if constexpr (std::is_base_of_v, PassT>) { + Passes.push_back(std::unique_ptr( + new MachinePassModelT(std::forward(Pass)))); + } else if constexpr (std::is_same_v>) { + for (auto &P : Pass.Passes) + Passes.push_back(std::move(P)); + } else { + Passes.push_back(std::unique_ptr( + new PassModelT(std::forward(Pass)))); + } +} + template <> PreservedAnalyses PassManager::run(MachineFunction &, diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h index ec8b809d40bf..108465478d37 100644 --- a/llvm/include/llvm/IR/PassManager.h +++ b/llvm/include/llvm/IR/PassManager.h @@ -267,29 +267,24 @@ public: return PA; } - template - LLVM_ATTRIBUTE_MINSIZE - std::enable_if_t::value> - addPass(PassT &&Pass) { + // FIXME: Revert to enable_if style when gcc >= 11.1 + template LLVM_ATTRIBUTE_MINSIZE void addPass(PassT &&Pass) { using PassModelT = detail::PassModel; - // Do not use make_unique or emplace_back, they cause too many template - // instantiations, causing terrible compile times. - Passes.push_back(std::unique_ptr( - new PassModelT(std::forward(Pass)))); - } - - /// When adding a pass manager pass that has the same type as this pass - /// manager, simply move the passes over. This is because we don't have use - /// cases rely on executing nested pass managers. Doing this could reduce - /// implementation complexity and avoid potential invalidation issues that may - /// happen with nested pass managers of the same type. - template - LLVM_ATTRIBUTE_MINSIZE - std::enable_if_t::value> - addPass(PassT &&Pass) { - for (auto &P : Pass.Passes) - Passes.push_back(std::move(P)); + if constexpr (!std::is_same_v) { + // Do not use make_unique or emplace_back, they cause too many template + // instantiations, causing terrible compile times. + Passes.push_back(std::unique_ptr( + new PassModelT(std::forward(Pass)))); + } else { + /// When adding a pass manager pass that has the same type as this pass + /// manager, simply move the passes over. This is because we don't have + /// use cases rely on executing nested pass managers. Doing this could + /// reduce implementation complexity and avoid potential invalidation + /// issues that may happen with nested pass managers of the same type. + for (auto &P : Pass.Passes) + Passes.push_back(std::move(P)); + } } /// Returns if the pass manager contains any passes. diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index 016602730e0e..2f77ae655d9b 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -127,6 +127,8 @@ MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass()) // MACHINE_FUNCTION_PASS("free-machine-function", FreeMachineFunctionPass()) MACHINE_FUNCTION_PASS("no-op-machine-function", NoOpMachineFunctionPass()) MACHINE_FUNCTION_PASS("print", PrintMIRPass()) +MACHINE_FUNCTION_PASS("require-all-machine-function-properties", + RequireAllMachineFunctionPropertiesPass()) #undef MACHINE_FUNCTION_PASS // After a pass is converted to new pass manager, its entry should be moved from diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index f60f4eb3f0ef..57975e34d426 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -365,6 +365,33 @@ public: static StringRef name() { return "TriggerVerifierErrorPass"; } }; +// A pass requires all MachineFunctionProperties. +// DO NOT USE THIS EXCEPT FOR TESTING! +class RequireAllMachineFunctionPropertiesPass + : public MachinePassInfoMixin { +public: + PreservedAnalyses run(MachineFunction &, MachineFunctionAnalysisManager &) { + return PreservedAnalyses::none(); + } + + static MachineFunctionProperties getRequiredProperties() { + MachineFunctionProperties MFProps; + MFProps.set(MachineFunctionProperties::Property::FailedISel); + MFProps.set(MachineFunctionProperties::Property::FailsVerification); + MFProps.set(MachineFunctionProperties::Property::IsSSA); + MFProps.set(MachineFunctionProperties::Property::Legalized); + MFProps.set(MachineFunctionProperties::Property::NoPHIs); + MFProps.set(MachineFunctionProperties::Property::NoVRegs); + MFProps.set(MachineFunctionProperties::Property::RegBankSelected); + MFProps.set(MachineFunctionProperties::Property::Selected); + MFProps.set(MachineFunctionProperties::Property::TiedOpsRewritten); + MFProps.set(MachineFunctionProperties::Property::TracksDebugUserValues); + MFProps.set(MachineFunctionProperties::Property::TracksLiveness); + return MFProps; + } + static StringRef name() { return "RequireAllMachineFunctionPropertiesPass"; } +}; + } // namespace PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO, diff --git a/llvm/test/tools/llc/new-pm/machine-function-properties.mir b/llvm/test/tools/llc/new-pm/machine-function-properties.mir new file mode 100644 index 000000000000..a9eb88ec6988 --- /dev/null +++ b/llvm/test/tools/llc/new-pm/machine-function-properties.mir @@ -0,0 +1,12 @@ +# REQUIRES: asserts +# RUN: not --crash llc -mtriple=x86_64-pc-linux-gnu -passes=require-all-machine-function-properties -filetype=null %s 2>&1 | FileCheck %s + +# CHECK: MachineFunctionProperties required by RequireAllMachineFunctionPropertiesPass pass are not met by function f. + +--- +name: f +selected: false +body: | + bb.0: + RET 0 +... -- GitLab From 29e8bfc13c6078ed07e6474e8c9634c42aa2f6f4 Mon Sep 17 00:00:00 2001 From: Brandon Wu Date: Sat, 30 Mar 2024 21:05:33 +0800 Subject: [PATCH 0050/1690] [RISCV] RISCV vector calling convention (2/2) (#79096) This commit handles vector arguments/return for function definition/call, the new class RVVArgDispatcher is added for doing all vector register assignment including mask types, data types as well as tuple types. It precomputes the register number for each argument as per https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#standard-vector-calling-convention-variant and it's passed to calling convention function to handle all vector arguments. Depends on: #78550 --- .../Target/RISCV/GISel/RISCVCallLowering.cpp | 55 +++--- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 182 ++++++++++++++---- llvm/lib/Target/RISCV/RISCVISelLowering.h | 56 +++++- llvm/test/CodeGen/RISCV/rvv/calling-conv.ll | 87 +++++++++ .../RISCV/rvv/vector-deinterleave-load.ll | 6 +- .../CodeGen/RISCV/rvv/vector-deinterleave.ll | 19 +- 6 files changed, 322 insertions(+), 83 deletions(-) diff --git a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp index 45e19cdea300..8af4bc658409 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp +++ b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp @@ -34,14 +34,15 @@ private: // Whether this is assigning args for a return. bool IsRet; - // true if assignArg has been called for a mask argument, false otherwise. - bool AssignedFirstMaskArg = false; + RVVArgDispatcher &RVVDispatcher; public: RISCVOutgoingValueAssigner( - RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet) + RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet, + RVVArgDispatcher &RVVDispatcher) : CallLowering::OutgoingValueAssigner(nullptr), - RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet) {} + RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet), + RVVDispatcher(RVVDispatcher) {} bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, @@ -51,16 +52,9 @@ public: const DataLayout &DL = MF.getDataLayout(); const RISCVSubtarget &Subtarget = MF.getSubtarget(); - std::optional FirstMaskArgument; - if (Subtarget.hasVInstructions() && !AssignedFirstMaskArg && - ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1) { - FirstMaskArgument = ValNo; - AssignedFirstMaskArg = true; - } - if (RISCVAssignFn(DL, Subtarget.getTargetABI(), ValNo, ValVT, LocVT, LocInfo, Flags, State, Info.IsFixed, IsRet, Info.Ty, - *Subtarget.getTargetLowering(), FirstMaskArgument)) + *Subtarget.getTargetLowering(), RVVDispatcher)) return true; StackSize = State.getStackSize(); @@ -181,14 +175,15 @@ private: // Whether this is assigning args from a return. bool IsRet; - // true if assignArg has been called for a mask argument, false otherwise. - bool AssignedFirstMaskArg = false; + RVVArgDispatcher &RVVDispatcher; public: RISCVIncomingValueAssigner( - RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet) + RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet, + RVVArgDispatcher &RVVDispatcher) : CallLowering::IncomingValueAssigner(nullptr), - RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet) {} + RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet), + RVVDispatcher(RVVDispatcher) {} bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, @@ -201,16 +196,9 @@ public: if (LocVT.isScalableVector()) MF.getInfo()->setIsVectorCall(); - std::optional FirstMaskArgument; - if (Subtarget.hasVInstructions() && !AssignedFirstMaskArg && - ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1) { - FirstMaskArgument = ValNo; - AssignedFirstMaskArg = true; - } - if (RISCVAssignFn(DL, Subtarget.getTargetABI(), ValNo, ValVT, LocVT, LocInfo, Flags, State, /*IsFixed=*/true, IsRet, Info.Ty, - *Subtarget.getTargetLowering(), FirstMaskArgument)) + *Subtarget.getTargetLowering(), RVVDispatcher)) return true; StackSize = State.getStackSize(); @@ -420,9 +408,11 @@ bool RISCVCallLowering::lowerReturnVal(MachineIRBuilder &MIRBuilder, SmallVector SplitRetInfos; splitToValueTypes(OrigRetInfo, SplitRetInfos, DL, CC); + RVVArgDispatcher Dispatcher{&MF, getTLI(), + F.getReturnType()}; RISCVOutgoingValueAssigner Assigner( CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV, - /*IsRet=*/true); + /*IsRet=*/true, Dispatcher); RISCVOutgoingValueHandler Handler(MIRBuilder, MF.getRegInfo(), Ret); return determineAndHandleAssignments(Handler, Assigner, SplitRetInfos, MIRBuilder, CC, F.isVarArg()); @@ -531,6 +521,7 @@ bool RISCVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, CallingConv::ID CC = F.getCallingConv(); SmallVector SplitArgInfos; + SmallVector TypeList; unsigned Index = 0; for (auto &Arg : F.args()) { // Construct the ArgInfo object from destination register and argument type. @@ -542,12 +533,15 @@ bool RISCVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, // correspondingly and appended to SplitArgInfos. splitToValueTypes(AInfo, SplitArgInfos, DL, CC); + TypeList.push_back(Arg.getType()); + ++Index; } + RVVArgDispatcher Dispatcher{&MF, getTLI(), TypeList}; RISCVIncomingValueAssigner Assigner( CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV, - /*IsRet=*/false); + /*IsRet=*/false, Dispatcher); RISCVFormalArgHandler Handler(MIRBuilder, MF.getRegInfo()); SmallVector ArgLocs; @@ -585,11 +579,13 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, SmallVector SplitArgInfos; SmallVector Outs; + SmallVector TypeList; for (auto &AInfo : Info.OrigArgs) { // Handle any required unmerging of split value types from a given VReg into // physical registers. ArgInfo objects are constructed correspondingly and // appended to SplitArgInfos. splitToValueTypes(AInfo, SplitArgInfos, DL, CC); + TypeList.push_back(AInfo.Ty); } // TODO: Support tail calls. @@ -607,9 +603,10 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); Call.addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv)); + RVVArgDispatcher ArgDispatcher{&MF, getTLI(), TypeList}; RISCVOutgoingValueAssigner ArgAssigner( CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV, - /*IsRet=*/false); + /*IsRet=*/false, ArgDispatcher); RISCVOutgoingValueHandler ArgHandler(MIRBuilder, MF.getRegInfo(), Call); if (!determineAndHandleAssignments(ArgHandler, ArgAssigner, SplitArgInfos, MIRBuilder, CC, Info.IsVarArg)) @@ -637,9 +634,11 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, SmallVector SplitRetInfos; splitToValueTypes(Info.OrigRet, SplitRetInfos, DL, CC); + RVVArgDispatcher RetDispatcher{&MF, getTLI(), + F.getReturnType()}; RISCVIncomingValueAssigner RetAssigner( CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV, - /*IsRet=*/true); + /*IsRet=*/true, RetDispatcher); RISCVCallReturnHandler RetHandler(MIRBuilder, MF.getRegInfo(), Call); if (!determineAndHandleAssignments(RetHandler, RetAssigner, SplitRetInfos, MIRBuilder, CC, Info.IsVarArg)) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index e48ca4a905ce..f693cbd3bea5 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -22,6 +22,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/VectorUtils.h" +#include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" @@ -18057,33 +18058,12 @@ static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, return false; } -static unsigned allocateRVVReg(MVT ValVT, unsigned ValNo, - std::optional FirstMaskArgument, - CCState &State, const RISCVTargetLowering &TLI) { - const TargetRegisterClass *RC = TLI.getRegClassFor(ValVT); - if (RC == &RISCV::VRRegClass) { - // Assign the first mask argument to V0. - // This is an interim calling convention and it may be changed in the - // future. - if (FirstMaskArgument && ValNo == *FirstMaskArgument) - return State.AllocateReg(RISCV::V0); - return State.AllocateReg(ArgVRs); - } - if (RC == &RISCV::VRM2RegClass) - return State.AllocateReg(ArgVRM2s); - if (RC == &RISCV::VRM4RegClass) - return State.AllocateReg(ArgVRM4s); - if (RC == &RISCV::VRM8RegClass) - return State.AllocateReg(ArgVRM8s); - llvm_unreachable("Unhandled register class for ValueType"); -} - // Implements the RISC-V calling convention. Returns true upon failure. bool RISCV::CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, - std::optional FirstMaskArgument) { + RVVArgDispatcher &RVVDispatcher) { unsigned XLen = DL.getLargestLegalIntTypeSizeInBits(); assert(XLen == 32 || XLen == 64); MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64; @@ -18252,7 +18232,7 @@ bool RISCV::CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, else if (ValVT == MVT::f64 && !UseGPRForF64) Reg = State.AllocateReg(ArgFPR64s); else if (ValVT.isVector()) { - Reg = allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI); + Reg = RVVDispatcher.getNextPhysReg(); if (!Reg) { // For return values, the vector must be passed fully via registers or // via the stack. @@ -18338,9 +18318,13 @@ void RISCVTargetLowering::analyzeInputArgs( unsigned NumArgs = Ins.size(); FunctionType *FType = MF.getFunction().getFunctionType(); - std::optional FirstMaskArgument; - if (Subtarget.hasVInstructions()) - FirstMaskArgument = preAssignMask(Ins); + SmallVector TypeList; + if (IsRet) + TypeList.push_back(MF.getFunction().getReturnType()); + else + for (const Argument &Arg : MF.getFunction().args()) + TypeList.push_back(Arg.getType()); + RVVArgDispatcher Dispatcher{&MF, this, TypeList}; for (unsigned i = 0; i != NumArgs; ++i) { MVT ArgVT = Ins[i].VT; @@ -18355,7 +18339,7 @@ void RISCVTargetLowering::analyzeInputArgs( RISCVABI::ABI ABI = MF.getSubtarget().getTargetABI(); if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy, *this, - FirstMaskArgument)) { + Dispatcher)) { LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type " << ArgVT << '\n'); llvm_unreachable(nullptr); @@ -18369,9 +18353,13 @@ void RISCVTargetLowering::analyzeOutputArgs( CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const { unsigned NumArgs = Outs.size(); - std::optional FirstMaskArgument; - if (Subtarget.hasVInstructions()) - FirstMaskArgument = preAssignMask(Outs); + SmallVector TypeList; + if (IsRet) + TypeList.push_back(MF.getFunction().getReturnType()); + else if (CLI) + for (const TargetLowering::ArgListEntry &Arg : CLI->getArgs()) + TypeList.push_back(Arg.Ty); + RVVArgDispatcher Dispatcher{&MF, this, TypeList}; for (unsigned i = 0; i != NumArgs; i++) { MVT ArgVT = Outs[i].VT; @@ -18381,7 +18369,7 @@ void RISCVTargetLowering::analyzeOutputArgs( RISCVABI::ABI ABI = MF.getSubtarget().getTargetABI(); if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *this, - FirstMaskArgument)) { + Dispatcher)) { LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type " << ArgVT << "\n"); llvm_unreachable(nullptr); @@ -18562,7 +18550,7 @@ bool RISCV::CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, - std::optional FirstMaskArgument) { + RVVArgDispatcher &RVVDispatcher) { if (LocVT == MVT::i32 || LocVT == MVT::i64) { if (unsigned Reg = State.AllocateReg(getFastCCArgGPRs(ABI))) { State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); @@ -18640,13 +18628,14 @@ bool RISCV::CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, } if (LocVT.isVector()) { - if (unsigned Reg = - allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI)) { + MCPhysReg AllocatedVReg = RVVDispatcher.getNextPhysReg(); + if (AllocatedVReg) { // Fixed-length vectors are located in the corresponding scalable-vector // container types. if (ValVT.isFixedLengthVector()) LocVT = TLI.getContainerForFixedLengthVector(LocVT); - State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); + State.addLoc( + CCValAssign::getReg(ValNo, ValVT, AllocatedVReg, LocVT, LocInfo)); } else { // Try and pass the address via a "fast" GPR. if (unsigned GPRReg = State.AllocateReg(getFastCCArgGPRs(ABI))) { @@ -19274,17 +19263,15 @@ bool RISCVTargetLowering::CanLowerReturn( SmallVector RVLocs; CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); - std::optional FirstMaskArgument; - if (Subtarget.hasVInstructions()) - FirstMaskArgument = preAssignMask(Outs); + RVVArgDispatcher Dispatcher{&MF, this, MF.getFunction().getReturnType()}; for (unsigned i = 0, e = Outs.size(); i != e; ++i) { MVT VT = Outs[i].VT; ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; RISCVABI::ABI ABI = MF.getSubtarget().getTargetABI(); if (RISCV::CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full, - ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr, - *this, FirstMaskArgument)) + ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, + nullptr, *this, Dispatcher)) return false; } return true; @@ -21081,6 +21068,121 @@ unsigned RISCVTargetLowering::getMinimumJumpTableEntries() const { return Subtarget.getMinimumJumpTableEntries(); } +void RVVArgDispatcher::constructArgInfos(ArrayRef TypeList) { + const DataLayout &DL = MF->getDataLayout(); + const Function &F = MF->getFunction(); + LLVMContext &Context = F.getContext(); + + bool FirstVMaskAssigned = false; + for (Type *Ty : TypeList) { + StructType *STy = dyn_cast(Ty); + if (STy && STy->containsHomogeneousScalableVectorTypes()) { + Type *ElemTy = STy->getTypeAtIndex(0U); + EVT VT = TLI->getValueType(DL, ElemTy); + MVT RegisterVT = + TLI->getRegisterTypeForCallingConv(Context, F.getCallingConv(), VT); + + RVVArgInfos.push_back({STy->getNumElements(), RegisterVT, false}); + } else { + SmallVector ValueVTs; + ComputeValueVTs(*TLI, DL, Ty, ValueVTs); + + for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues; + ++Value) { + EVT VT = ValueVTs[Value]; + MVT RegisterVT = + TLI->getRegisterTypeForCallingConv(Context, F.getCallingConv(), VT); + unsigned NumRegs = + TLI->getNumRegistersForCallingConv(Context, F.getCallingConv(), VT); + + // Skip non-RVV register type + if (!RegisterVT.isVector()) + continue; + + if (RegisterVT.isFixedLengthVector()) + RegisterVT = TLI->getContainerForFixedLengthVector(RegisterVT); + + if (!FirstVMaskAssigned && + RegisterVT.getVectorElementType() == MVT::i1) { + RVVArgInfos.push_back({1, RegisterVT, true}); + FirstVMaskAssigned = true; + } else { + RVVArgInfos.push_back({1, RegisterVT, false}); + } + + RVVArgInfos.insert(RVVArgInfos.end(), --NumRegs, + {1, RegisterVT, false}); + } + } + } +} + +void RVVArgDispatcher::allocatePhysReg(unsigned NF, unsigned LMul, + unsigned StartReg) { + assert((StartReg % LMul) == 0 && + "Start register number should be multiple of lmul"); + const MCPhysReg *VRArrays; + switch (LMul) { + default: + report_fatal_error("Invalid lmul"); + case 1: + VRArrays = ArgVRs; + break; + case 2: + VRArrays = ArgVRM2s; + break; + case 4: + VRArrays = ArgVRM4s; + break; + case 8: + VRArrays = ArgVRM8s; + break; + } + + for (unsigned i = 0; i < NF; ++i) + if (StartReg) + AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]); + else + AllocatedPhysRegs.push_back(MCPhysReg()); +} + +/// This function determines if each RVV argument is passed by register, if the +/// argument can be assigned to a VR, then give it a specific register. +/// Otherwise, assign the argument to 0 which is a invalid MCPhysReg. +void RVVArgDispatcher::compute() { + uint32_t AssignedMap = 0; + auto allocate = [&](const RVVArgInfo &ArgInfo) { + // Allocate first vector mask argument to V0. + if (ArgInfo.FirstVMask) { + AllocatedPhysRegs.push_back(RISCV::V0); + return; + } + + unsigned RegsNeeded = divideCeil( + ArgInfo.VT.getSizeInBits().getKnownMinValue(), RISCV::RVVBitsPerBlock); + unsigned TotalRegsNeeded = ArgInfo.NF * RegsNeeded; + for (unsigned StartReg = 0; StartReg + TotalRegsNeeded <= NumArgVRs; + StartReg += RegsNeeded) { + uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg; + if ((AssignedMap & Map) == 0) { + allocatePhysReg(ArgInfo.NF, RegsNeeded, StartReg + 8); + AssignedMap |= Map; + return; + } + } + + allocatePhysReg(ArgInfo.NF, RegsNeeded, 0); + }; + + for (unsigned i = 0; i < RVVArgInfos.size(); ++i) + allocate(RVVArgInfos[i]); +} + +MCPhysReg RVVArgDispatcher::getNextPhysReg() { + assert(CurIdx < AllocatedPhysRegs.size() && "Index out of range"); + return AllocatedPhysRegs[CurIdx++]; +} + namespace llvm::RISCVVIntrinsicsTable { #define GET_RISCVVIntrinsicsTable_IMPL diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index ace5b3fd2b95..c28552354bf4 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -24,6 +24,7 @@ namespace llvm { class InstructionCost; class RISCVSubtarget; struct RISCVRegisterInfo; +class RVVArgDispatcher; namespace RISCVISD { // clang-format off @@ -875,7 +876,7 @@ public: ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, - std::optional FirstMaskArgument); + RVVArgDispatcher &RVVDispatcher); private: void analyzeInputArgs(MachineFunction &MF, CCState &CCInfo, @@ -1015,19 +1016,68 @@ private: unsigned getMinimumJumpTableEntries() const override; }; +/// As per the spec, the rules for passing vector arguments are as follows: +/// +/// 1. For the first vector mask argument, use v0 to pass it. +/// 2. For vector data arguments or rest vector mask arguments, starting from +/// the v8 register, if a vector register group between v8-v23 that has not been +/// allocated can be found and the first register number is a multiple of LMUL, +/// then allocate this vector register group to the argument and mark these +/// registers as allocated. Otherwise, pass it by reference and are replaced in +/// the argument list with the address. +/// 3. For tuple vector data arguments, starting from the v8 register, if +/// NFIELDS consecutive vector register groups between v8-v23 that have not been +/// allocated can be found and the first register number is a multiple of LMUL, +/// then allocate these vector register groups to the argument and mark these +/// registers as allocated. Otherwise, pass it by reference and are replaced in +/// the argument list with the address. +class RVVArgDispatcher { +public: + static constexpr unsigned NumArgVRs = 16; + + struct RVVArgInfo { + unsigned NF; + MVT VT; + bool FirstVMask = false; + }; + + RVVArgDispatcher(const MachineFunction *MF, const RISCVTargetLowering *TLI, + ArrayRef TypeList) + : MF(MF), TLI(TLI) { + constructArgInfos(TypeList); + compute(); + } + + MCPhysReg getNextPhysReg(); + +private: + SmallVector RVVArgInfos; + SmallVector AllocatedPhysRegs; + + const MachineFunction *MF = nullptr; + const RISCVTargetLowering *TLI = nullptr; + + unsigned CurIdx = 0; + + void constructArgInfos(ArrayRef TypeList); + void compute(); + void allocatePhysReg(unsigned NF = 1, unsigned LMul = 1, + unsigned StartReg = 0); +}; + namespace RISCV { bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, - std::optional FirstMaskArgument); + RVVArgDispatcher &RVVDispatcher); bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, - std::optional FirstMaskArgument); + RVVArgDispatcher &RVVDispatcher); bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, diff --git a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll index 78e8700a9fef..90edb994ce82 100644 --- a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll +++ b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll @@ -162,3 +162,90 @@ define void @caller_tuple_argument({, } %x) } declare void @callee_tuple_argument({, }) + +; %0 -> v8 +; %1 -> v9 +define @case1( %0, %1) { +; CHECK-LABEL: case1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vadd.vv v8, v8, v9 +; CHECK-NEXT: ret + %a = add %0, %1 + ret %a +} + +; %0 -> v8 +; %1 -> v10-v11 +; %2 -> v9 +define @case2_1( %0, %1, %2) { +; CHECK-LABEL: case2_1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vadd.vv v8, v8, v9 +; CHECK-NEXT: ret + %a = add %0, %2 + ret %a +} +define @case2_2( %0, %1, %2) { +; CHECK-LABEL: case2_2: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vadd.vv v8, v10, v10 +; CHECK-NEXT: ret + %a = add %1, %1 + ret %a +} + +; %0 -> v8 +; %1 -> {v10-v11, v12-v13} +; %2 -> v9 +define @case3_1( %0, {, } %1, %2) { +; CHECK-LABEL: case3_1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vadd.vv v8, v8, v9 +; CHECK-NEXT: ret + %add = add %0, %2 + ret %add +} +define @case3_2( %0, {, } %1, %2) { +; CHECK-LABEL: case3_2: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: ret + %a = extractvalue { , } %1, 0 + %b = extractvalue { , } %1, 1 + %add = add %a, %b + ret %add +} + +; %0 -> v8 +; %1 -> {by-ref, by-ref} +; %2 -> v9 +define @case4_1( %0, {, } %1, %2) { +; CHECK-LABEL: case4_1: +; CHECK: # %bb.0: +; CHECK-NEXT: csrr a1, vlenb +; CHECK-NEXT: slli a1, a1, 3 +; CHECK-NEXT: add a1, a0, a1 +; CHECK-NEXT: vl8re64.v v8, (a1) +; CHECK-NEXT: vl8re64.v v16, (a0) +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vadd.vv v8, v16, v8 +; CHECK-NEXT: ret + %a = extractvalue { , } %1, 0 + %b = extractvalue { , } %1, 1 + %add = add %a, %b + ret %add +} +define @case4_2( %0, {, } %1, %2) { +; CHECK-LABEL: case4_2: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vadd.vv v8, v8, v9 +; CHECK-NEXT: ret + %add = add %0, %2 + ret %add +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll index a320aecc6fce..6a712080fda7 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll @@ -18,10 +18,10 @@ define {, } @vector_deinterleave_load_nxv16i ; CHECK-NEXT: vmerge.vim v14, v10, 1, v0 ; CHECK-NEXT: vmv1r.v v0, v8 ; CHECK-NEXT: vmerge.vim v12, v10, 1, v0 -; CHECK-NEXT: vnsrl.wi v8, v12, 0 -; CHECK-NEXT: vmsne.vi v0, v8, 0 -; CHECK-NEXT: vnsrl.wi v10, v12, 8 +; CHECK-NEXT: vnsrl.wi v10, v12, 0 ; CHECK-NEXT: vmsne.vi v8, v10, 0 +; CHECK-NEXT: vnsrl.wi v10, v12, 8 +; CHECK-NEXT: vmsne.vi v9, v10, 0 ; CHECK-NEXT: ret %vec = load , ptr %p %retval = call {, } @llvm.experimental.vector.deinterleave2.nxv32i1( %vec) diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll index ef4baf34d23f..d98597fabcd9 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll @@ -8,18 +8,18 @@ define {, } @vector_deinterleave_nxv16i1_nxv ; CHECK-LABEL: vector_deinterleave_nxv16i1_nxv32i1: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e8, m2, ta, ma -; CHECK-NEXT: vmv.v.i v10, 0 -; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vmv.v.i v8, 0 +; CHECK-NEXT: vmerge.vim v12, v8, 1, v0 ; CHECK-NEXT: csrr a0, vlenb ; CHECK-NEXT: srli a0, a0, 2 ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vslidedown.vx v0, v0, a0 ; CHECK-NEXT: vsetvli a0, zero, e8, m2, ta, ma -; CHECK-NEXT: vmerge.vim v10, v10, 1, v0 -; CHECK-NEXT: vnsrl.wi v12, v8, 0 -; CHECK-NEXT: vmsne.vi v0, v12, 0 -; CHECK-NEXT: vnsrl.wi v12, v8, 8 -; CHECK-NEXT: vmsne.vi v8, v12, 0 +; CHECK-NEXT: vmerge.vim v14, v8, 1, v0 +; CHECK-NEXT: vnsrl.wi v10, v12, 0 +; CHECK-NEXT: vmsne.vi v8, v10, 0 +; CHECK-NEXT: vnsrl.wi v10, v12, 8 +; CHECK-NEXT: vmsne.vi v9, v10, 0 ; CHECK-NEXT: ret %retval = call {, } @llvm.experimental.vector.deinterleave2.nxv32i1( %vec) ret {, } %retval @@ -102,12 +102,13 @@ define {, } @vector_deinterleave_nxv64i1_nxv ; CHECK-NEXT: vsetvli a0, zero, e8, m4, ta, ma ; CHECK-NEXT: vnsrl.wi v28, v8, 0 ; CHECK-NEXT: vsetvli a0, zero, e8, m8, ta, ma -; CHECK-NEXT: vmsne.vi v0, v24, 0 +; CHECK-NEXT: vmsne.vi v7, v24, 0 ; CHECK-NEXT: vsetvli a0, zero, e8, m4, ta, ma ; CHECK-NEXT: vnsrl.wi v24, v16, 8 ; CHECK-NEXT: vnsrl.wi v28, v8, 8 ; CHECK-NEXT: vsetvli a0, zero, e8, m8, ta, ma -; CHECK-NEXT: vmsne.vi v8, v24, 0 +; CHECK-NEXT: vmsne.vi v9, v24, 0 +; CHECK-NEXT: vmv1r.v v8, v7 ; CHECK-NEXT: ret %retval = call {, } @llvm.experimental.vector.deinterleave2.nxv128i1( %vec) ret {, } %retval -- GitLab From 37c175af955f0aeab67e8c553a0a47b2ed0fdba2 Mon Sep 17 00:00:00 2001 From: Jacek Caban Date: Sat, 30 Mar 2024 16:43:33 +0100 Subject: [PATCH 0051/1690] [clang][Driver] Pass -machine argument to the linker explicitly for ARM64EC targets. (#86835) This is required by the linker. Also add a new -marm64x command line argument to allow specifying -machine:arm64x. --- clang/include/clang/Driver/Options.td | 3 +++ clang/lib/Driver/ToolChains/MSVC.cpp | 8 ++++++++ clang/test/Driver/msvc-link.c | 20 ++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 8a3929eeb669..04eb87f0d5d1 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -4510,6 +4510,9 @@ def mwindows : Joined<["-"], "mwindows">, Group; def mdll : Joined<["-"], "mdll">, Group; def municode : Joined<["-"], "municode">, Group; def mthreads : Joined<["-"], "mthreads">, Group; +def marm64x : Joined<["-"], "marm64x">, Group, + Visibility<[ClangOption, CLOption]>, + HelpText<"Link as a hybrid ARM64X image">; def mguard_EQ : Joined<["-"], "mguard=">, Group, HelpText<"Enable or disable Control Flow Guard checks and guard tables emission">, Values<"none,cf,cf-nochecks">; diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp index dc534a33e6d0..fbf2f45b5438 100644 --- a/clang/lib/Driver/ToolChains/MSVC.cpp +++ b/clang/lib/Driver/ToolChains/MSVC.cpp @@ -79,6 +79,11 @@ void visualstudio::Linker::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back( Args.MakeArgString(std::string("-out:") + Output.getFilename())); + if (Args.hasArg(options::OPT_marm64x)) + CmdArgs.push_back("-machine:arm64x"); + else if (TC.getTriple().isWindowsArm64EC()) + CmdArgs.push_back("-machine:arm64ec"); + if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles) && !C.getDriver().IsCLMode() && !C.getDriver().IsFlangMode()) { CmdArgs.push_back("-defaultlib:libcmt"); @@ -1017,4 +1022,7 @@ void MSVCToolChain::addClangTargetOptions( if (DriverArgs.hasFlag(options::OPT_fno_rtti, options::OPT_frtti, /*Default=*/false)) CC1Args.push_back("-D_HAS_STATIC_RTTI=0"); + + if (Arg *A = DriverArgs.getLastArgNoClaim(options::OPT_marm64x)) + A->ignoreTargetSpecific(); } diff --git a/clang/test/Driver/msvc-link.c b/clang/test/Driver/msvc-link.c index 64e099ea6304..f80b043c6cfc 100644 --- a/clang/test/Driver/msvc-link.c +++ b/clang/test/Driver/msvc-link.c @@ -36,3 +36,23 @@ // VFSOVERLAY: "--vfsoverlay" // VFSOVERLAY: lld-link // VFSOVERLAY: "/vfsoverlay:{{.*}}" "{{.*}}.obj" + +// RUN: %clang -target arm64ec-pc-windows-msvc -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=ARM64EC %s +// RUN: %clang_cl -target arm64ec-pc-windows-msvc -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=ARM64EC %s +// RUN: %clang_cl -arm64EC -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=ARM64EC %s +// ARM64EC: "-machine:arm64ec" + +// RUN: %clang -target arm64ec-pc-windows-msvc -fuse-ld=link -marm64x -### %s 2>&1 | \ +// RUN: FileCheck --check-prefix=ARM64X %s +// RUN: %clang -target aarch64-pc-windows-msvc -fuse-ld=link -marm64x -### %s 2>&1 | \ +// RUN: FileCheck --check-prefix=ARM64X %s +// RUN: %clang_cl -marm64x -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=ARM64X %s +// RUN: %clang_cl -arm64EC -marm64x -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=ARM64X %s +// ARM64X: "-machine:arm64x" + +// RUN: not %clang -target x86_64-linux-gnu -marm64x -### %s 2>&1 | FileCheck --check-prefix=HYBRID-ERR %s +// HYBRID-ERR: error: unsupported option '-marm64x' for target 'x86_64-linux-gnu' + +// RUN: %clang -c -marm64x -target arm64ec-pc-windows-msvc -fuse-ld=link -### %s 2>&1 | \ +// RUN: FileCheck --check-prefix=HYBRID-WARN %s +// HYBRID-WARN: warning: argument unused during compilation: '-marm64x' [-Wunused-command-line-argument] -- GitLab From 799e1d6a128438268bbf64b8c30ad3722f55524b Mon Sep 17 00:00:00 2001 From: Jacek Caban Date: Sat, 30 Mar 2024 16:48:39 +0100 Subject: [PATCH 0052/1690] [IR] Use EXPORTAS for ARM64EC mangled symbols with dllexport attribute. (#81940) We currently just use mangled name. This works fine, because linker should detect that and demangle it for the export table. However, on MSVC, the compiler is more specific and passes demangled name as well, with EXPORTAS. This PR aims to match that. MSVC doesn't use quotes in this case, so I added '#' to the list of characters that don't need it. --- llvm/lib/IR/Mangler.cpp | 14 ++++++++- llvm/test/CodeGen/AArch64/dllexport.ll | 39 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/llvm/lib/IR/Mangler.cpp b/llvm/lib/IR/Mangler.cpp index 3acac2c3e3db..c4e8f005d6c8 100644 --- a/llvm/lib/IR/Mangler.cpp +++ b/llvm/lib/IR/Mangler.cpp @@ -18,8 +18,10 @@ #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/Module.h" +#include "llvm/Object/COFF.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TargetParser/Triple.h" + using namespace llvm; namespace { @@ -192,7 +194,7 @@ void Mangler::getNameWithPrefix(SmallVectorImpl &OutName, // Check if the name needs quotes to be safe for the linker to interpret. static bool canBeUnquotedInDirective(char C) { - return isAlnum(C) || C == '_' || C == '@'; + return isAlnum(C) || C == '_' || C == '@' || C == '#'; } static bool canBeUnquotedInDirective(StringRef Name) { @@ -233,6 +235,16 @@ void llvm::emitLinkerFlagsForGlobalCOFF(raw_ostream &OS, const GlobalValue *GV, } else { Mangler.getNameWithPrefix(OS, GV, false); } + if (TT.isWindowsArm64EC()) { + // Use EXPORTAS for mangled ARM64EC symbols. + // FIXME: During LTO, we're invoked prior to the EC lowering pass, + // so symbols are not yet mangled. Emitting the unmangled name + // typically functions correctly; the linker can resolve the export + // with the demangled alias. + if (std::optional demangledName = + object::getArm64ECDemangledFunctionName(GV->getName())) + OS << ",EXPORTAS," << *demangledName; + } if (NeedQuotes) OS << "\""; diff --git a/llvm/test/CodeGen/AArch64/dllexport.ll b/llvm/test/CodeGen/AArch64/dllexport.ll index 81ba674a0ded..580fb5fd9e79 100644 --- a/llvm/test/CodeGen/AArch64/dllexport.ll +++ b/llvm/test/CodeGen/AArch64/dllexport.ll @@ -1,5 +1,7 @@ ; RUN: llc -mtriple aarch64-windows-gnu -filetype asm -o - %s | FileCheck %s -check-prefix CHECK -check-prefix CHECK-GNU ; RUN: llc -mtriple aarch64-windows-msvc -filetype asm -o - %s | FileCheck %s -check-prefix CHECK -check-prefix CHECK-MSVC +; RUN: llc -mtriple arm64ec-windows-gnu -filetype asm -o - %s | FileCheck %s -check-prefix CHECK -check-prefix CHECK-GNU-EC +; RUN: llc -mtriple arm64ec-windows-msvc -filetype asm -o - %s | FileCheck %s -check-prefix CHECK -check-prefix CHECK-MSVC-EC define void @f() { ret void @@ -71,3 +73,40 @@ define weak_odr dllexport void @l() { ; CHECK-MSVC: .ascii " /EXPORT:s" ; CHECK-MSVC: .ascii " /EXPORT:t" ; CHECK-MSVC: .ascii " /EXPORT:u" + +; CHECK-GNU-EC-NOT: -export:f +; CHECK-GNU-EC-NOT: -export:#f,EXPORTAS,f +; CHECK-GNU-EC: .ascii " -export:#g,EXPORTAS,g +; CHECK-GNU-EC: .ascii " -export:#h,EXPORTAS,h +; CHECK-GNU-EC-NOT: -export:i +; CHECK-GNU-EC-NOT: -export:#i,EXPORTAS,i +; CHECK-GNU-EC: .ascii " -export:#j,EXPORTAS,j" +; CHECK-GNU-EC: .ascii " -export:#k,EXPORTAS,k" +; CHECK-GNU-EC: .ascii " -export:#l,EXPORTAS,l" +; CHECK-GNU-EC: .ascii " -export:m,data" +; CHECK-GNU-EC: .ascii " -export:n,data" +; CHECK-GNU-EC: .ascii " -export:o,data" +; CHECK-GNU-EC: .ascii " -export:p,data" +; CHECK-GNU-EC: .ascii " -export:q,data" +; CHECK-GNU-EC: .ascii " -export:r" +; CHECK-GNU-EC: .ascii " -export:s" +; CHECK-GNU-EC: .ascii " -export:t" +; CHECK-GNU-EC: .ascii " -export:u" +; CHECK-MSVC-EC-NOT: /EXPORT:f +; CHECK-MSVC-EC-NOT: /EXPORT:#f,EXPORTAS,f +; CHECK-MSVC-EC: .ascii " /EXPORT:#g,EXPORTAS,g" +; CHECK-MSVC-EC: .ascii " /EXPORT:#h,EXPORTAS,h" +; CHECK-MSVC-EC-NOT: /EXPORT:i +; CHECK-MSVC-EC-NOT: /EXPORT:#i,EXPORTAS,i +; CHECK-MSVC-EC: .ascii " /EXPORT:#j,EXPORTAS,j" +; CHECK-MSVC-EC: .ascii " /EXPORT:#k,EXPORTAS,k" +; CHECK-MSVC-EC: .ascii " /EXPORT:#l,EXPORTAS,l" +; CHECK-MSVC-EC: .ascii " /EXPORT:m,DATA" +; CHECK-MSVC-EC: .ascii " /EXPORT:n,DATA" +; CHECK-MSVC-EC: .ascii " /EXPORT:o,DATA" +; CHECK-MSVC-EC: .ascii " /EXPORT:p,DATA" +; CHECK-MSVC-EC: .ascii " /EXPORT:q,DATA" +; CHECK-MSVC-EC: .ascii " /EXPORT:r" +; CHECK-MSVC-EC: .ascii " /EXPORT:s" +; CHECK-MSVC-EC: .ascii " /EXPORT:t" +; CHECK-MSVC-EC: .ascii " /EXPORT:u" -- GitLab From 556bf03a572fb8f6a91f1ad5227603b37717c55d Mon Sep 17 00:00:00 2001 From: Jacek Caban Date: Sat, 30 Mar 2024 17:18:12 +0100 Subject: [PATCH 0053/1690] [clang][NFC] Fix ARM64EC clang-cl linker driver tests. (#87160) Add '--' argument to clang-cl to avoid interpreting input files with /U option. Fix for llvm-clang-aarch64-darwin buildbot failure after #86835. --- clang/test/Driver/msvc-link.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clang/test/Driver/msvc-link.c b/clang/test/Driver/msvc-link.c index f80b043c6cfc..b5c32b173785 100644 --- a/clang/test/Driver/msvc-link.c +++ b/clang/test/Driver/msvc-link.c @@ -38,16 +38,16 @@ // VFSOVERLAY: "/vfsoverlay:{{.*}}" "{{.*}}.obj" // RUN: %clang -target arm64ec-pc-windows-msvc -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=ARM64EC %s -// RUN: %clang_cl -target arm64ec-pc-windows-msvc -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=ARM64EC %s -// RUN: %clang_cl -arm64EC -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=ARM64EC %s +// RUN: %clang_cl -target arm64ec-pc-windows-msvc -fuse-ld=link -### -- %s 2>&1 | FileCheck --check-prefix=ARM64EC %s +// RUN: %clang_cl -arm64EC -fuse-ld=link -### -- %s 2>&1 | FileCheck --check-prefix=ARM64EC %s // ARM64EC: "-machine:arm64ec" // RUN: %clang -target arm64ec-pc-windows-msvc -fuse-ld=link -marm64x -### %s 2>&1 | \ // RUN: FileCheck --check-prefix=ARM64X %s // RUN: %clang -target aarch64-pc-windows-msvc -fuse-ld=link -marm64x -### %s 2>&1 | \ // RUN: FileCheck --check-prefix=ARM64X %s -// RUN: %clang_cl -marm64x -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=ARM64X %s -// RUN: %clang_cl -arm64EC -marm64x -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=ARM64X %s +// RUN: %clang_cl -marm64x -fuse-ld=link -### -- %s 2>&1 | FileCheck --check-prefix=ARM64X %s +// RUN: %clang_cl -arm64EC -marm64x -fuse-ld=link -### -- %s 2>&1 | FileCheck --check-prefix=ARM64X %s // ARM64X: "-machine:arm64x" // RUN: not %clang -target x86_64-linux-gnu -marm64x -### %s 2>&1 | FileCheck --check-prefix=HYBRID-ERR %s -- GitLab From 6aa53888a8e8a6e3f0bd279539703f4d4701b4e7 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Sat, 30 Mar 2024 17:50:59 +0100 Subject: [PATCH 0054/1690] [libc++][test] Improves src include paths. (#86864) Instead of including a relative path use an absolute path based on the available lit substitution. This makes it easier to understand what is included and moving the test to a different directory level no longer breaks the test. This is based on a question by @EricWF in https://github.com/llvm/llvm-project/pull/82113. --- .../directory_entry.mods/last_write_time.pass.cpp | 2 +- .../libcxx/input.output/filesystems/convert_file_time.pass.cpp | 2 +- libcxx/test/libcxx/time/time.zone/time.zone.db/rules.pass.cpp | 2 +- libcxx/test/libcxx/time/time.zone/time.zone.db/zones.pass.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp b/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp index 26703f748d87..1acbed55d2b5 100644 --- a/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp +++ b/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp @@ -9,7 +9,7 @@ // UNSUPPORTED: c++03, c++11, c++14 // UNSUPPORTED: availability-filesystem-missing // UNSUPPORTED: no-filesystem -// ADDITIONAL_COMPILE_FLAGS: -I %S/../../../../../../src +// ADDITIONAL_COMPILE_FLAGS: -I %{libcxx-dir}/src // This test relies on calling functions from the libcxx internal headers // of ; the Windows implementation uses different diff --git a/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp b/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp index 3c901c4e9f2e..699bbfb9e6dc 100644 --- a/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp +++ b/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp @@ -13,7 +13,7 @@ // typedef TrivialClock file_time_type; -// ADDITIONAL_COMPILE_FLAGS: -I %S/../../../../src -Wno-macro-redefined +// ADDITIONAL_COMPILE_FLAGS: -I %{libcxx-dir}/src -Wno-macro-redefined #include #include diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.db/rules.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.db/rules.pass.cpp index 4814f4aad87f..fcfc34625fbe 100644 --- a/libcxx/test/libcxx/time/time.zone/time.zone.db/rules.pass.cpp +++ b/libcxx/test/libcxx/time/time.zone/time.zone.db/rules.pass.cpp @@ -17,7 +17,7 @@ // Tests the IANA database rules parsing and operations. // This is not part of the public tzdb interface. // The test uses private implementation headers. -// ADDITIONAL_COMPILE_FLAGS: -I %S/../../../../../src/include +// ADDITIONAL_COMPILE_FLAGS: -I %{libcxx-dir}/src/include #include #include diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.db/zones.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.db/zones.pass.cpp index 8571e0d05ebb..e97b36fca2bb 100644 --- a/libcxx/test/libcxx/time/time.zone/time.zone.db/zones.pass.cpp +++ b/libcxx/test/libcxx/time/time.zone/time.zone.db/zones.pass.cpp @@ -17,7 +17,7 @@ // Tests the IANA database zones parsing and operations. // This is not part of the public tzdb interface. // The test uses private implementation headers. -// ADDITIONAL_COMPILE_FLAGS: -I %S/../../../../../src/include +// ADDITIONAL_COMPILE_FLAGS: -I %{libcxx-dir}/src/include #include #include -- GitLab From 3c8ede9f4524fd8defbbf41788003a6d4b0afb04 Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Sat, 30 Mar 2024 21:33:56 +0100 Subject: [PATCH 0055/1690] [HLSL][clang] Move hlsl_wave_get_lane_index to EmitHLSLBuiltinExpr (#87131) Resolves #87109 --- clang/lib/CodeGen/CGBuiltin.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 287e763bad82..bb007231c0b7 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -5892,16 +5892,6 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Name), {NDRange, Kernel, Block})); } - - case Builtin::BI__builtin_hlsl_wave_get_lane_index: { - auto *CI = EmitRuntimeCall(CGM.CreateRuntimeFunction( - llvm::FunctionType::get(IntTy, {}, false), "__hlsl_wave_get_lane_index", - {}, false, true)); - if (getTarget().getTriple().isSPIRVLogical()) - CI = dyn_cast(addControlledConvergenceToken(CI)); - return RValue::get(CI); - } - case Builtin::BI__builtin_store_half: case Builtin::BI__builtin_store_halff: { Value *Val = EmitScalarExpr(E->getArg(0)); @@ -18317,6 +18307,14 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, /*ReturnType=*/Op0->getType(), Intrinsic::dx_rsqrt, ArrayRef{Op0}, nullptr, "dx.rsqrt"); } + case Builtin::BI__builtin_hlsl_wave_get_lane_index: { + auto *CI = EmitRuntimeCall(CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, {}, false), "__hlsl_wave_get_lane_index", + {}, false, true)); + if (getTarget().getTriple().isSPIRVLogical()) + CI = dyn_cast(addControlledConvergenceToken(CI)); + return CI; + } } return nullptr; } -- GitLab From 8d9cb6b016c9dce8412c329277f1bbaa3f16961c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 30 Mar 2024 20:14:07 +0000 Subject: [PATCH 0056/1690] [VPlan] Inline getVPValue in only caller (NFCI). --- llvm/lib/Transforms/Vectorize/VPlan.h | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 272a42e44d88..6c90c793e66c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3013,15 +3013,6 @@ public: Value2VPValue[V] = VPV; } - /// Returns the VPValue for \p V. - VPValue *getVPValue(Value *V) { - assert(V && "Trying to get the VPValue of a null Value"); - assert(Value2VPValue.count(V) && "Value does not exist in VPlan"); - assert(Value2VPValue[V]->isLiveIn() && - "Only live-ins should be in mapping"); - return Value2VPValue[V]; - } - /// Gets the live-in VPValue for \p V or adds a new live-in (if none exists /// yet) for \p V. VPValue *getOrAddLiveIn(Value *V) { @@ -3032,7 +3023,10 @@ public: addVPValue(V, VPV); } - return getVPValue(V); + assert(Value2VPValue.count(V) && "Value does not exist in VPlan"); + assert(Value2VPValue[V]->isLiveIn() && + "Only live-ins should be in mapping"); + return Value2VPValue[V]; } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -- GitLab From a67b9326cd0448072a1192951f12f3927f31af8c Mon Sep 17 00:00:00 2001 From: Min Hsu Date: Sat, 30 Mar 2024 15:18:03 -0700 Subject: [PATCH 0057/1690] [M68k][NFC] Refactoring memory operands of different sizes with foreach I'm planning to add memory operands that produce floating point type. In order to prevent code bloating induced by memory operands of not just integer but floating point types, I factored the sizes from these operands with foreach loops. --- llvm/lib/Target/M68k/M68kInstrInfo.td | 116 +++++++++++--------------- 1 file changed, 51 insertions(+), 65 deletions(-) diff --git a/llvm/lib/Target/M68k/M68kInstrInfo.td b/llvm/lib/Target/M68k/M68kInstrInfo.td index 84eb8e56da76..3a9e4fe77f1c 100644 --- a/llvm/lib/Target/M68k/M68kInstrInfo.td +++ b/llvm/lib/Target/M68k/M68kInstrInfo.td @@ -200,32 +200,35 @@ class MxRegOp; -def MxXRD32 : MxRegOp; - -def MxXRD16_TC : MxRegOp; -def MxXRD32_TC : MxRegOp; +foreach size = [16, 32] in { + def MxXRD # size : MxRegOp("i"#size), !cast("XR"#size), + !cast("MxSize"#size), "r">; + def MxXRD # size # _TC : MxRegOp("i"#size), !cast("XR"#size#"_TC"), + !cast("MxSize"#size), "r">; +} // foreach size // DATA REGISTER DIRECT. The operand is in the data register specified by // the effective address register field. let ParserMatchClass = MxDRegClass in { - def MxDRD8 : MxRegOp; - def MxDRD16 : MxRegOp; - def MxDRD32 : MxRegOp; - - def MxDRD16_TC : MxRegOp; - def MxDRD32_TC : MxRegOp; -} +foreach size = [8, 16, 32] in { + def MxDRD # size : MxRegOp("i"#size), !cast("DR"#size), + !cast("MxSize"#size), "d">; + if !gt(size, 8) then + def MxDRD # size # _TC : MxRegOp("i"#size), !cast("DR"#size#"_TC"), + !cast("MxSize"#size), "d">; +} // foreach size +} // let ParserMatchClass // ADDRESS REGISTER DIRECT. The operand is in the address register specified by // the effective address register field. let ParserMatchClass = MxARegClass in { - def MxARD16 : MxRegOp; - def MxARD32 : MxRegOp; - - def MxARD16_TC : MxRegOp; - def MxARD32_TC : MxRegOp; -} +foreach size = [16, 32] in { + def MxARD # size : MxRegOp("i"#size), !cast("AR"#size), + !cast("MxSize"#size), "a">; + def MxARD # size # _TC : MxRegOp("i"#size), !cast("AR"#size#"_TC"), + !cast("MxSize"#size), "a">; +} // foreach size +} // let ParserMatchClass // FLOATING POINT DATA REGISTER. let ParserMatchClass = MxFPDRegClass in { @@ -255,13 +258,11 @@ class MxMemOp; -def MxARI8 : MxMemOp<(ops AR32), MxSize8, "j", "printARI8Mem", MxARI>; -def MxARI16 : MxMemOp<(ops AR32), MxSize16, "j", "printARI16Mem", MxARI>; -def MxARI32 : MxMemOp<(ops AR32), MxSize32, "j", "printARI32Mem", MxARI>; - -def MxARI8_TC : MxMemOp<(ops AR32_TC), MxSize8, "j", "printARI8Mem", MxARI>; -def MxARI16_TC : MxMemOp<(ops AR32_TC), MxSize16, "j", "printARI16Mem", MxARI>; -def MxARI32_TC : MxMemOp<(ops AR32_TC), MxSize32, "j", "printARI32Mem", MxARI>; +foreach size = ["8", "16", "32"] in { + defvar ResSize = !cast("MxSize"#size); + def MxARI # size : MxMemOp<(ops AR32), ResSize, "j", "printARI"#size#"Mem", MxARI>; + def MxARI # size # _TC : MxMemOp<(ops AR32_TC), ResSize, "j", "printARI"#size#"Mem", MxARI>; +} // foreach size // ADDRESS REGISTER INDIRECT WITH POSTINCREMENT. The address of the operand is // in the address register specified by the register field. After the operand @@ -271,13 +272,11 @@ def MxARI32_TC : MxMemOp<(ops AR32_TC), MxSize32, "j", "printARI32Mem", MxARI // by two rather than one to keep the stack pointer on a word boundary. // The reference is classified as a data reference. def MxARIPI : MxOpClass<"ARIPI">; -def MxARIPI8 : MxMemOp<(ops AR32), MxSize8, "o", "printARIPI8Mem", MxARIPI>; -def MxARIPI16 : MxMemOp<(ops AR32), MxSize16, "o", "printARIPI16Mem", MxARIPI>; -def MxARIPI32 : MxMemOp<(ops AR32), MxSize32, "o", "printARIPI32Mem", MxARIPI>; - -def MxARIPI8_TC : MxMemOp<(ops AR32_TC), MxSize8, "o", "printARIPI8Mem", MxARIPI>; -def MxARIPI16_TC : MxMemOp<(ops AR32_TC), MxSize16, "o", "printARIPI16Mem", MxARIPI>; -def MxARIPI32_TC : MxMemOp<(ops AR32_TC), MxSize32, "o", "printARIPI32Mem", MxARIPI>; +foreach size = ["8", "16", "32"] in { + defvar ResSize = !cast("MxSize"#size); + def MxARIPI # size : MxMemOp<(ops AR32), ResSize, "o", "printARIPI"#size#"Mem", MxARIPI>; + def MxARIPI # size # _TC : MxMemOp<(ops AR32_TC), ResSize, "o", "printARIPI"#size#"Mem", MxARIPI>; +} // foreach size // ADDRESS REGISTER INDIRECT WITH PREDECREMENT. The address of the operand is in // the address register specified by the register field. Before the operand @@ -287,13 +286,11 @@ def MxARIPI32_TC : MxMemOp<(ops AR32_TC), MxSize32, "o", "printARIPI32Mem", MxA // two rather than one to keep the stack pointer on a word boundary. // The reference is classified as a data reference. def MxARIPD : MxOpClass<"ARIPD">; -def MxARIPD8 : MxMemOp<(ops AR32), MxSize8, "e", "printARIPD8Mem", MxARIPD>; -def MxARIPD16 : MxMemOp<(ops AR32), MxSize16, "e", "printARIPD16Mem", MxARIPD>; -def MxARIPD32 : MxMemOp<(ops AR32), MxSize32, "e", "printARIPD32Mem", MxARIPD>; - -def MxARIPD8_TC : MxMemOp<(ops AR32_TC), MxSize8, "e", "printARIPD8Mem", MxARIPD>; -def MxARIPD16_TC : MxMemOp<(ops AR32_TC), MxSize16, "e", "printARIPD16Mem", MxARIPD>; -def MxARIPD32_TC : MxMemOp<(ops AR32_TC), MxSize32, "e", "printARIPD32Mem", MxARIPD>; +foreach size = ["8", "16", "32"] in { + defvar ResSize = !cast("MxSize"#size); + def MxARIPD # size : MxMemOp<(ops AR32), ResSize, "e", "printARIPD"#size#"Mem", MxARIPD>; + def MxARIPD # size # _TC : MxMemOp<(ops AR32_TC), ResSize, "e", "printARIPD"#size#"Mem", MxARIPD>; +} // foreach size // ADDRESS REGISTER INDIRECT WITH DISPLACEMENT. This addressing mode requires one // word of extension. The address of the operand is the sum of the address in @@ -301,13 +298,11 @@ def MxARIPD32_TC : MxMemOp<(ops AR32_TC), MxSize32, "e", "printARIPD32Mem", MxA // extension word. The reference is classified as a data reference with the // exception of the jump and jump-to-subroutine instructions. def MxARID : MxOpClass<"ARID">; -def MxARID8 : MxMemOp<(ops i16imm:$disp, AR32:$reg), MxSize8, "p", "printARID8Mem", MxARID>; -def MxARID16 : MxMemOp<(ops i16imm:$disp, AR32:$reg), MxSize16, "p", "printARID16Mem", MxARID>; -def MxARID32 : MxMemOp<(ops i16imm:$disp, AR32:$reg), MxSize32, "p", "printARID32Mem", MxARID>; - -def MxARID8_TC : MxMemOp<(ops i16imm:$disp, AR32_TC:$reg), MxSize8, "p", "printARID8Mem", MxARID>; -def MxARID16_TC : MxMemOp<(ops i16imm:$disp, AR32_TC:$reg), MxSize16, "p", "printARID16Mem", MxARID>; -def MxARID32_TC : MxMemOp<(ops i16imm:$disp, AR32_TC:$reg), MxSize32, "p", "printARID32Mem", MxARID>; +foreach size = ["8", "16", "32"] in { + defvar ResSize = !cast("MxSize"#size); + def MxARID # size : MxMemOp<(ops i16imm:$disp, AR32:$reg), ResSize, "p", "printARID"#size#"Mem", MxARID>; + def MxARID # size # _TC : MxMemOp<(ops i16imm:$disp, AR32_TC:$reg), ResSize, "p", "printARID"#size#"Mem", MxARID>; +} // foreach size // ADDRESS REGISTER INDIRECT WITH INDEX. This addressing mode requires one word // of extension. The address of the operand is the sum of the address in the @@ -316,19 +311,11 @@ def MxARID32_TC : MxMemOp<(ops i16imm:$disp, AR32_TC:$reg), MxSize32, "p", "pr // The reference is classified as a data reference with the exception of the // jump and jump-to-subroutine instructions def MxARII : MxOpClass<"ARII">; -def MxARII8 : MxMemOp<(ops i8imm:$disp, AR32:$reg, XR32:$index), - MxSize8, "f", "printARII8Mem", MxARII>; -def MxARII16 : MxMemOp<(ops i8imm:$disp, AR32:$reg, XR32:$index), - MxSize16, "f", "printARII16Mem", MxARII>; -def MxARII32 : MxMemOp<(ops i8imm:$disp, AR32:$reg, XR32:$index), - MxSize32, "f", "printARII32Mem", MxARII>; - -def MxARII8_TC : MxMemOp<(ops i8imm:$disp, AR32_TC:$reg, XR32_TC:$index), - MxSize8, "f", "printARII8Mem", MxARII>; -def MxARII16_TC : MxMemOp<(ops i8imm:$disp, AR32_TC:$reg, XR32_TC:$index), - MxSize16, "f", "printARII16Mem", MxARII>; -def MxARII32_TC : MxMemOp<(ops i8imm:$disp, AR32_TC:$reg, XR32_TC:$index), - MxSize32, "f", "printARII32Mem", MxARII>; +foreach size = ["8", "16", "32"] in { + defvar ResSize = !cast("MxSize"#size); + def MxARII # size : MxMemOp<(ops i8imm:$disp, AR32:$reg, XR32:$index), ResSize, "f", "printARII"#size#"Mem", MxARII>; + def MxARII # size # _TC : MxMemOp<(ops i8imm:$disp, AR32_TC:$reg, XR32:$index), ResSize, "f", "printARII"#size#"Mem", MxARII>; +} // foreach size // ABSOLUTE SHORT ADDRESS. This addressing mode requires one word of extension. // The address of the operand is the extension word. The 16-bit address is sign @@ -361,14 +348,14 @@ def MxPCD : MxOpClass<"PCD">; def MxPCI : MxOpClass<"PCI">; let OperandType = "OPERAND_PCREL" in { +foreach size = ["8", "16", "32"] in { +defvar ResSize = !cast("MxSize"#size); // PROGRAM COUNTER WITH DISPLACEMENT. This addressing mode requires one word of // extension. The address of the operand is the sum of the address in the program // counter and the Sign-extended 16-bit displacement integer in the extension // word. The value in the program counter is the address of the extension word. // The reference is classified as a program reference. -def MxPCD8 : MxMemOp<(ops i16imm), MxSize8, "q", "printPCD8Mem", MxPCD>; -def MxPCD16 : MxMemOp<(ops i16imm), MxSize16, "q", "printPCD16Mem", MxPCD>; -def MxPCD32 : MxMemOp<(ops i16imm), MxSize32, "q", "printPCD32Mem", MxPCD>; +def MxPCD # size : MxMemOp<(ops i16imm), ResSize, "q", "printPCD"#size#"Mem", MxPCD>; // PROGRAM COUNTER WITH INDEX. This addressing mode requires one word of // extension. The address is the sum of the address in the program counter, the @@ -376,9 +363,8 @@ def MxPCD32 : MxMemOp<(ops i16imm), MxSize32, "q", "printPCD32Mem", MxPCD>; // word, and the contents of the index register. The value in the program // counter is the address of the extension word. This reference is classified as // a program reference. -def MxPCI8 : MxMemOp<(ops i8imm:$disp, XR32:$index), MxSize8, "k", "printPCI8Mem", MxPCI>; -def MxPCI16 : MxMemOp<(ops i8imm:$disp, XR32:$index), MxSize16, "k", "printPCI16Mem", MxPCI>; -def MxPCI32 : MxMemOp<(ops i8imm:$disp, XR32:$index), MxSize32, "k", "printPCI32Mem", MxPCI>; +def MxPCI # size : MxMemOp<(ops i8imm:$disp, XR32:$index), ResSize, "k", "printPCI"#size#"Mem", MxPCI>; +} // foreach size } // OPERAND_PCREL def MxImm : AsmOperandClass { -- GitLab From c0febca3a64530c1cc3f117aec6a28c37114e572 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Sat, 30 Mar 2024 20:43:23 -0700 Subject: [PATCH 0058/1690] [BOLT][NFC] Refactor BC::createBinaryContext for #81346 (#87172) --- bolt/include/bolt/Core/BinaryContext.h | 3 +- bolt/lib/Core/BinaryContext.cpp | 39 ++++++++++++----------- bolt/lib/Rewrite/DWARFRewriter.cpp | 2 +- bolt/lib/Rewrite/MachORewriteInstance.cpp | 35 ++------------------ bolt/lib/Rewrite/RewriteInstance.cpp | 19 ++++++++++- bolt/unittests/Core/BinaryContext.cpp | 4 +-- bolt/unittests/Core/MCPlusBuilder.cpp | 4 +-- 7 files changed, 48 insertions(+), 58 deletions(-) diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h index 741b1a36af86..8b1af9e81539 100644 --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -265,7 +265,8 @@ class BinaryContext { public: static Expected> - createBinaryContext(const ObjectFile *File, bool IsPIC, + createBinaryContext(Triple TheTriple, StringRef InputFileName, + SubtargetFeatures *Features, bool IsPIC, std::unique_ptr DwCtx, JournalingStreams Logger); diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp index 267f43f65e20..7b474bc40dac 100644 --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -162,28 +162,30 @@ BinaryContext::~BinaryContext() { /// Create BinaryContext for a given architecture \p ArchName and /// triple \p TripleName. -Expected> -BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC, - std::unique_ptr DwCtx, - JournalingStreams Logger) { +Expected> BinaryContext::createBinaryContext( + Triple TheTriple, StringRef InputFileName, SubtargetFeatures *Features, + bool IsPIC, std::unique_ptr DwCtx, JournalingStreams Logger) { StringRef ArchName = ""; std::string FeaturesStr = ""; - switch (File->getArch()) { + switch (TheTriple.getArch()) { case llvm::Triple::x86_64: + if (Features) + return createFatalBOLTError( + "x86_64 target does not use SubtargetFeatures"); ArchName = "x86-64"; FeaturesStr = "+nopl"; break; case llvm::Triple::aarch64: + if (Features) + return createFatalBOLTError( + "AArch64 target does not use SubtargetFeatures"); ArchName = "aarch64"; FeaturesStr = "+all"; break; case llvm::Triple::riscv64: { ArchName = "riscv64"; - Expected Features = File->getFeatures(); - - if (auto E = Features.takeError()) - return std::move(E); - + if (!Features) + return createFatalBOLTError("RISCV target needs SubtargetFeatures"); // We rely on relaxation for some transformations (e.g., promoting all calls // to PseudoCALL and then making JITLink relax them). Since the relax // feature is not stored in the object file, we manually enable it. @@ -196,12 +198,11 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC, "BOLT-ERROR: Unrecognized machine in ELF file"); } - auto TheTriple = std::make_unique(File->makeTriple()); - const std::string TripleName = TheTriple->str(); + const std::string TripleName = TheTriple.str(); std::string Error; const Target *TheTarget = - TargetRegistry::lookupTarget(std::string(ArchName), *TheTriple, Error); + TargetRegistry::lookupTarget(std::string(ArchName), TheTriple, Error); if (!TheTarget) return createStringError(make_error_code(std::errc::not_supported), Twine("BOLT-ERROR: ", Error)); @@ -240,13 +241,13 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC, Twine("BOLT-ERROR: no instruction info for target ", TripleName)); std::unique_ptr Ctx( - new MCContext(*TheTriple, AsmInfo.get(), MRI.get(), STI.get())); + new MCContext(TheTriple, AsmInfo.get(), MRI.get(), STI.get())); std::unique_ptr MOFI( TheTarget->createMCObjectFileInfo(*Ctx, IsPIC)); Ctx->setObjectFileInfo(MOFI.get()); // We do not support X86 Large code model. Change this in the future. bool Large = false; - if (TheTriple->getArch() == llvm::Triple::aarch64) + if (TheTriple.getArch() == llvm::Triple::aarch64) Large = true; unsigned LSDAEncoding = Large ? dwarf::DW_EH_PE_absptr : dwarf::DW_EH_PE_udata4; @@ -273,7 +274,7 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC, int AsmPrinterVariant = AsmInfo->getAssemblerDialect(); std::unique_ptr InstructionPrinter( - TheTarget->createMCInstPrinter(*TheTriple, AsmPrinterVariant, *AsmInfo, + TheTarget->createMCInstPrinter(TheTriple, AsmPrinterVariant, *AsmInfo, *MII, *MRI)); if (!InstructionPrinter) return createStringError( @@ -285,8 +286,8 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC, TheTarget->createMCCodeEmitter(*MII, *Ctx)); auto BC = std::make_unique( - std::move(Ctx), std::move(DwCtx), std::move(TheTriple), TheTarget, - std::string(TripleName), std::move(MCE), std::move(MOFI), + std::move(Ctx), std::move(DwCtx), std::make_unique(TheTriple), + TheTarget, std::string(TripleName), std::move(MCE), std::move(MOFI), std::move(AsmInfo), std::move(MII), std::move(STI), std::move(InstructionPrinter), std::move(MIA), nullptr, std::move(MRI), std::move(DisAsm), Logger); @@ -296,7 +297,7 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC, BC->MAB = std::unique_ptr( BC->TheTarget->createMCAsmBackend(*BC->STI, *BC->MRI, MCTargetOptions())); - BC->setFilename(File->getFileName()); + BC->setFilename(InputFileName); BC->HasFixedLoadAddress = !IsPIC; diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp index 601a2105fc26..6a6f5708cdf7 100644 --- a/bolt/lib/Rewrite/DWARFRewriter.cpp +++ b/bolt/lib/Rewrite/DWARFRewriter.cpp @@ -1685,7 +1685,7 @@ namespace { std::unique_ptr createDwarfOnlyBC(const object::ObjectFile &File) { return cantFail(BinaryContext::createBinaryContext( - &File, false, + File.makeTriple(), File.getFileName(), nullptr, false, DWARFContext::create(File, DWARFContext::ProcessDebugRelocations::Ignore, nullptr, "", WithColor::defaultErrorHandler, WithColor::defaultWarningHandler), diff --git a/bolt/lib/Rewrite/MachORewriteInstance.cpp b/bolt/lib/Rewrite/MachORewriteInstance.cpp index 0970a0507ebe..172cb640bf91 100644 --- a/bolt/lib/Rewrite/MachORewriteInstance.cpp +++ b/bolt/lib/Rewrite/MachORewriteInstance.cpp @@ -18,6 +18,7 @@ #include "bolt/Rewrite/BinaryPassManager.h" #include "bolt/Rewrite/ExecutableFileMemoryManager.h" #include "bolt/Rewrite/JITLinkLinker.h" +#include "bolt/Rewrite/RewriteInstance.h" #include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h" #include "bolt/Utils/Utils.h" #include "llvm/MC/MCObjectStreamer.h" @@ -54,37 +55,6 @@ extern cl::opt Verbosity; namespace llvm { namespace bolt { -extern MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *, - const MCInstrInfo *, - const MCRegisterInfo *, - const MCSubtargetInfo *); -extern MCPlusBuilder *createAArch64MCPlusBuilder(const MCInstrAnalysis *, - const MCInstrInfo *, - const MCRegisterInfo *, - const MCSubtargetInfo *); - -namespace { - -MCPlusBuilder *createMCPlusBuilder(const Triple::ArchType Arch, - const MCInstrAnalysis *Analysis, - const MCInstrInfo *Info, - const MCRegisterInfo *RegInfo, - const MCSubtargetInfo *STI) { -#ifdef X86_AVAILABLE - if (Arch == Triple::x86_64) - return createX86MCPlusBuilder(Analysis, Info, RegInfo, STI); -#endif - -#ifdef AARCH64_AVAILABLE - if (Arch == Triple::aarch64) - return createAArch64MCPlusBuilder(Analysis, Info, RegInfo, STI); -#endif - - llvm_unreachable("architecture unsupported by MCPlusBuilder"); -} - -} // anonymous namespace - #define DEBUG_TYPE "bolt" Expected> @@ -103,7 +73,8 @@ MachORewriteInstance::MachORewriteInstance(object::MachOObjectFile *InputFile, : InputFile(InputFile), ToolPath(ToolPath) { ErrorAsOutParameter EAO(&Err); auto BCOrErr = BinaryContext::createBinaryContext( - InputFile, /* IsPIC */ true, DWARFContext::create(*InputFile), + InputFile->makeTriple(), InputFile->getFileName(), nullptr, + /* IsPIC */ true, DWARFContext::create(*InputFile), {llvm::outs(), llvm::errs()}); if (Error E = BCOrErr.takeError()) { Err = std::move(E); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 5ca5594117a6..0c8ee0d41723 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -268,6 +268,10 @@ namespace bolt { extern const char *BoltRevision; +// Weird location for createMCPlusBuilder, but this is here to avoid a +// cyclic dependency of libCore (its natural place) and libTarget. libRewrite +// can depend on libTarget, but not libCore. Since libRewrite is the only +// user of this function, we define it here. MCPlusBuilder *createMCPlusBuilder(const Triple::ArchType Arch, const MCInstrAnalysis *Analysis, const MCInstrInfo *Info, @@ -345,8 +349,21 @@ RewriteInstance::RewriteInstance(ELFObjectFileBase *File, const int Argc, Stderr.SetUnbuffered(); LLVM_DEBUG(dbgs().SetUnbuffered()); + // Read RISCV subtarget features from input file + std::unique_ptr Features; + Triple TheTriple = File->makeTriple(); + if (TheTriple.getArch() == llvm::Triple::riscv64) { + Expected FeaturesOrErr = File->getFeatures(); + if (auto E = FeaturesOrErr.takeError()) { + Err = std::move(E); + return; + } else { + Features.reset(new SubtargetFeatures(*FeaturesOrErr)); + } + } + auto BCOrErr = BinaryContext::createBinaryContext( - File, IsPIC, + TheTriple, File->getFileName(), Features.get(), IsPIC, DWARFContext::create(*File, DWARFContext::ProcessDebugRelocations::Ignore, nullptr, opts::DWPPathName, WithColor::defaultErrorHandler, diff --git a/bolt/unittests/Core/BinaryContext.cpp b/bolt/unittests/Core/BinaryContext.cpp index 08619eccdd75..6737ba27bfcd 100644 --- a/bolt/unittests/Core/BinaryContext.cpp +++ b/bolt/unittests/Core/BinaryContext.cpp @@ -40,8 +40,8 @@ protected: void initializeBOLT() { BC = cantFail(BinaryContext::createBinaryContext( - ObjFile.get(), true, DWARFContext::create(*ObjFile.get()), - {llvm::outs(), llvm::errs()})); + ObjFile->makeTriple(), ObjFile->getFileName(), nullptr, true, + DWARFContext::create(*ObjFile.get()), {llvm::outs(), llvm::errs()})); ASSERT_FALSE(!BC); } diff --git a/bolt/unittests/Core/MCPlusBuilder.cpp b/bolt/unittests/Core/MCPlusBuilder.cpp index daf9f392b822..2fbfc792e42e 100644 --- a/bolt/unittests/Core/MCPlusBuilder.cpp +++ b/bolt/unittests/Core/MCPlusBuilder.cpp @@ -50,8 +50,8 @@ protected: void initializeBolt() { BC = cantFail(BinaryContext::createBinaryContext( - ObjFile.get(), true, DWARFContext::create(*ObjFile.get()), - {llvm::outs(), llvm::errs()})); + ObjFile->makeTriple(), ObjFile->getFileName(), nullptr, true, + DWARFContext::create(*ObjFile.get()), {llvm::outs(), llvm::errs()})); ASSERT_FALSE(!BC); BC->initializeTarget(std::unique_ptr( createMCPlusBuilder(GetParam(), BC->MIA.get(), BC->MII.get(), -- GitLab From ea92b1f9d0fc31f1fd97ad04eb0412003a37cb0d Mon Sep 17 00:00:00 2001 From: Karl-Johan Karlsson Date: Sun, 31 Mar 2024 11:09:52 +0200 Subject: [PATCH 0059/1690] [Sema] Implement support for -Wformat-signedness (#74440) In gcc there exist a modifier option -Wformat-signedness that turns on additional signedness warnings in the already existing -Wformat warning. This patch implements that support in clang. This is done by adding a dummy warning diag::warn_format_conversion_argument_type_mismatch_signedness that is never emitted and only used as an option to toggle the signedness warning in -Wformat. This will ensure gcc compatibility. --- clang/docs/ReleaseNotes.rst | 3 + clang/include/clang/AST/FormatString.h | 2 + clang/include/clang/Basic/DiagnosticGroups.td | 1 + .../clang/Basic/DiagnosticSemaKinds.td | 3 + clang/lib/AST/FormatString.cpp | 29 ++- clang/lib/Sema/SemaChecking.cpp | 33 ++- .../Sema/format-strings-signedness-fixit.c | 98 ++++++++ clang/test/Sema/format-strings-signedness.c | 222 ++++++++++++++++++ 8 files changed, 379 insertions(+), 12 deletions(-) create mode 100644 clang/test/Sema/format-strings-signedness-fixit.c create mode 100644 clang/test/Sema/format-strings-signedness.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 37b843915a0d..76eaf0bf11c3 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -310,6 +310,9 @@ Improvements to Clang's diagnostics - Clang now no longer diagnoses type definitions in ``offsetof`` in C23 mode. Fixes #GH83658. +- New ``-Wformat-signedness`` diagnostic that warn if the format string requires an + unsigned argument and the argument is signed and vice versa. + Improvements to Clang's time-trace ---------------------------------- diff --git a/clang/include/clang/AST/FormatString.h b/clang/include/clang/AST/FormatString.h index e2232fb4a471..a074dd23e2ad 100644 --- a/clang/include/clang/AST/FormatString.h +++ b/clang/include/clang/AST/FormatString.h @@ -284,6 +284,8 @@ public: /// The conversion specifier and the argument type are disallowed by the C /// standard, but are in practice harmless. For instance, "%p" and int*. NoMatchPedantic, + /// The conversion specifier and the argument type have different sign. + NoMatchSignedness, /// The conversion specifier and the argument type are compatible, but still /// seems likely to be an error. For instance, "%hd" and _Bool. NoMatchTypeConfusion, diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index 44035e2fd16f..520168f01fd8 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -985,6 +985,7 @@ def FormatSecurity : DiagGroup<"format-security">; def FormatNonStandard : DiagGroup<"format-non-iso">; def FormatY2K : DiagGroup<"format-y2k">; def FormatPedantic : DiagGroup<"format-pedantic">; +def FormatSignedness : DiagGroup<"format-signedness">; def FormatTypeConfusion : DiagGroup<"format-type-confusion">; def FormatOverflowNonKprintf: DiagGroup<"format-overflow-non-kprintf">; diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index fbc9c04dfba9..df57f5e6ce11 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9821,6 +9821,9 @@ def warn_format_conversion_argument_type_mismatch : Warning< def warn_format_conversion_argument_type_mismatch_pedantic : Extension< warn_format_conversion_argument_type_mismatch.Summary>, InGroup; +def warn_format_conversion_argument_type_mismatch_signedness : Warning< + warn_format_conversion_argument_type_mismatch.Summary>, + InGroup, DefaultIgnore; def warn_format_conversion_argument_type_mismatch_confusion : Warning< warn_format_conversion_argument_type_mismatch.Summary>, InGroup, DefaultIgnore; diff --git a/clang/lib/AST/FormatString.cpp b/clang/lib/AST/FormatString.cpp index 0c80ad109ccb..da8164bad518 100644 --- a/clang/lib/AST/FormatString.cpp +++ b/clang/lib/AST/FormatString.cpp @@ -413,7 +413,7 @@ ArgType::matchesType(ASTContext &C, QualType argTy) const { return Match; if (const auto *BT = argTy->getAs()) { // Check if the only difference between them is signed vs unsigned - // if true, we consider they are compatible. + // if true, return match signedness. switch (BT->getKind()) { default: break; @@ -423,44 +423,53 @@ ArgType::matchesType(ASTContext &C, QualType argTy) const { [[fallthrough]]; case BuiltinType::Char_S: case BuiltinType::SChar: + if (T == C.UnsignedShortTy || T == C.ShortTy) + return NoMatchTypeConfusion; + if (T == C.UnsignedCharTy) + return NoMatchSignedness; + if (T == C.SignedCharTy) + return Match; + break; case BuiltinType::Char_U: case BuiltinType::UChar: if (T == C.UnsignedShortTy || T == C.ShortTy) return NoMatchTypeConfusion; - if (T == C.UnsignedCharTy || T == C.SignedCharTy) + if (T == C.UnsignedCharTy) return Match; + if (T == C.SignedCharTy) + return NoMatchSignedness; break; case BuiltinType::Short: if (T == C.UnsignedShortTy) - return Match; + return NoMatchSignedness; break; case BuiltinType::UShort: if (T == C.ShortTy) - return Match; + return NoMatchSignedness; break; case BuiltinType::Int: if (T == C.UnsignedIntTy) - return Match; + return NoMatchSignedness; break; case BuiltinType::UInt: if (T == C.IntTy) - return Match; + return NoMatchSignedness; break; case BuiltinType::Long: if (T == C.UnsignedLongTy) - return Match; + return NoMatchSignedness; break; case BuiltinType::ULong: if (T == C.LongTy) - return Match; + return NoMatchSignedness; break; case BuiltinType::LongLong: if (T == C.UnsignedLongLongTy) - return Match; + return NoMatchSignedness; break; case BuiltinType::ULongLong: if (T == C.LongLongTy) - return Match; + return NoMatchSignedness; break; } // "Partially matched" because of promotions? diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 2684535d8e53..11401b6f56c0 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -12446,6 +12446,19 @@ isArithmeticArgumentPromotion(Sema &S, const ImplicitCastExpr *ICE) { S.Context.getFloatingTypeOrder(From, To) < 0; } +static analyze_format_string::ArgType::MatchKind +handleFormatSignedness(analyze_format_string::ArgType::MatchKind Match, + DiagnosticsEngine &Diags, SourceLocation Loc) { + if (Match == analyze_format_string::ArgType::NoMatchSignedness) { + Match = + Diags.isIgnored( + diag::warn_format_conversion_argument_type_mismatch_signedness, Loc) + ? analyze_format_string::ArgType::Match + : analyze_format_string::ArgType::NoMatch; + } + return Match; +} + bool CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS, const char *StartSpecifier, @@ -12489,6 +12502,9 @@ CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS, ArgType::MatchKind ImplicitMatch = ArgType::NoMatch; ArgType::MatchKind Match = AT.matchesType(S.Context, ExprTy); + ArgType::MatchKind OrigMatch = Match; + + Match = handleFormatSignedness(Match, S.getDiagnostics(), E->getExprLoc()); if (Match == ArgType::Match) return true; @@ -12512,6 +12528,14 @@ CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS, ICE->getType() == S.Context.UnsignedIntTy) { // All further checking is done on the subexpression ImplicitMatch = AT.matchesType(S.Context, ExprTy); + if (OrigMatch == ArgType::NoMatchSignedness && + ImplicitMatch != ArgType::NoMatchSignedness) + // If the original match was a signedness match this match on the + // implicit cast type also need to be signedness match otherwise we + // might introduce new unexpected warnings from -Wformat-signedness. + return true; + ImplicitMatch = handleFormatSignedness( + ImplicitMatch, S.getDiagnostics(), E->getExprLoc()); if (ImplicitMatch == ArgType::Match) return true; } @@ -12633,6 +12657,7 @@ CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS, case ArgType::Match: case ArgType::MatchPromotion: case ArgType::NoMatchPromotionTypeConfusion: + case ArgType::NoMatchSignedness: llvm_unreachable("expected non-matching"); case ArgType::NoMatchPedantic: Diag = diag::warn_format_conversion_argument_type_mismatch_pedantic; @@ -12668,8 +12693,10 @@ CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS, CastFix << (S.LangOpts.CPlusPlus ? ">" : ")"); SmallVector Hints; - if (AT.matchesType(S.Context, IntendedTy) != ArgType::Match || - ShouldNotPrintDirectly) + ArgType::MatchKind IntendedMatch = AT.matchesType(S.Context, IntendedTy); + IntendedMatch = handleFormatSignedness(IntendedMatch, S.getDiagnostics(), + E->getExprLoc()); + if ((IntendedMatch != ArgType::Match) || ShouldNotPrintDirectly) Hints.push_back(FixItHint::CreateReplacement(SpecRange, os.str())); if (const CStyleCastExpr *CCast = dyn_cast(E)) { @@ -12738,6 +12765,7 @@ CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS, case ArgType::Match: case ArgType::MatchPromotion: case ArgType::NoMatchPromotionTypeConfusion: + case ArgType::NoMatchSignedness: llvm_unreachable("expected non-matching"); case ArgType::NoMatchPedantic: Diag = diag::warn_format_conversion_argument_type_mismatch_pedantic; @@ -12949,6 +12977,7 @@ bool CheckScanfHandler::HandleScanfSpecifier( analyze_format_string::ArgType::MatchKind Match = AT.matchesType(S.Context, Ex->getType()); + Match = handleFormatSignedness(Match, S.getDiagnostics(), Ex->getExprLoc()); bool Pedantic = Match == analyze_format_string::ArgType::NoMatchPedantic; if (Match == analyze_format_string::ArgType::Match) return true; diff --git a/clang/test/Sema/format-strings-signedness-fixit.c b/clang/test/Sema/format-strings-signedness-fixit.c new file mode 100644 index 000000000000..b4e6e975657a --- /dev/null +++ b/clang/test/Sema/format-strings-signedness-fixit.c @@ -0,0 +1,98 @@ +// RUN: cp %s %t +// RUN: %clang_cc1 -triple=x86_64-pc-linux-gnu -Wformat -Wformat-signedness -fixit %t +// RUN: %clang_cc1 -triple=x86_64-pc-linux-gnu -fsyntax-only -Wformat -Wformat-signedness -Werror %t +// RUN: %clang_cc1 -triple=x86_64-pc-linux-gnu -E -o - %t | FileCheck %s + +#include + +int printf(const char *restrict format, ...); + +void test_printf_int(int x) +{ + printf("%u", x); +} + +void test_printf_unsigned(unsigned x) +{ + printf("%d", x); +} + +void test_printf_long(long x) +{ + printf("%lu", x); +} + +void test_printf_unsigned_long(unsigned long x) +{ + printf("%ld", x); +} + +void test_printf_long_long(long long x) +{ + printf("%llu", x); +} + +void test_printf_unsigned_long_long(unsigned long long x) +{ + printf("%lld", x); +} + +enum enum_int { + minus_1 = -1 +}; + +void test_printf_enum_int(enum enum_int x) +{ + printf("%u", x); +} + +enum enum_unsigned { + zero = 0 +}; + +void test_printf_enum_unsigned(enum enum_unsigned x) +{ + printf("%d", x); +} + +enum enum_long { + minus_one = -1, + int_val = INT_MAX, + unsigned_val = (unsigned)INT_MIN +}; + +void test_printf_enum_long(enum enum_long x) +{ + printf("%lu", x); +} + +enum enum_unsigned_long { + uint_max_plus = (unsigned long)UINT_MAX+1, +}; + +void test_printf_enum_unsigned_long(enum enum_unsigned_long x) +{ + printf("%ld", x); +} + +// Validate the fixes. +// CHECK: void test_printf_int(int x) +// CHECK: printf("%d", x); +// CHECK: void test_printf_unsigned(unsigned x) +// CHECK: printf("%u", x); +// CHECK: void test_printf_long(long x) +// CHECK: printf("%ld", x); +// CHECK: void test_printf_unsigned_long(unsigned long x) +// CHECK: printf("%lu", x); +// CHECK: void test_printf_long_long(long long x) +// CHECK: printf("%lld", x); +// CHECK: void test_printf_unsigned_long_long(unsigned long long x) +// CHECK: printf("%llu", x); +// CHECK: void test_printf_enum_int(enum enum_int x) +// CHECK: printf("%d", x); +// CHECK: void test_printf_enum_unsigned(enum enum_unsigned x) +// CHECK: printf("%u", x); +// CHECK: void test_printf_enum_long(enum enum_long x) +// CHECK: printf("%ld", x); +// CHECK: void test_printf_enum_unsigned_long(enum enum_unsigned_long x) +// CHECK: printf("%lu", x); diff --git a/clang/test/Sema/format-strings-signedness.c b/clang/test/Sema/format-strings-signedness.c new file mode 100644 index 000000000000..d5a8140d9ef8 --- /dev/null +++ b/clang/test/Sema/format-strings-signedness.c @@ -0,0 +1,222 @@ +// RUN: %clang_cc1 -triple=x86_64-pc-linux-gnu -std=c11 -fsyntax-only -verify -Wformat -Wformat-signedness %s +// RUN: %clang_cc1 -triple=x86_64-pc-win32 -std=c11 -fsyntax-only -verify -Wformat -Wformat-signedness %s + +// Verify that -Wformat-signedness alone (without -Wformat) trigger the +// warnings. Note in gcc this will not trigger the signedness warnings as +// -Wformat is default off in gcc. +// RUN: %clang_cc1 -triple=x86_64-pc-linux-gnu -std=c11 -fsyntax-only -verify -Wformat-signedness %s +// RUN: %clang_cc1 -triple=x86_64-pc-win32 -std=c11 -fsyntax-only -verify -Wformat-signedness %s + +// Verify that -Wformat-signedness warnings are not reported with only -Wformat +// (gcc compat). +// RUN: %clang_cc1 -triple=x86_64-pc-linux-gnu -std=c11 -fsyntax-only -Wformat -verify=okay %s + +// Verify that -Wformat-signedness with -Wno-format are not reported (gcc compat). +// RUN: %clang_cc1 -triple=x86_64-pc-linux-gnu -std=c11 -fsyntax-only -Wformat-signedness -Wno-format -verify=okay %s +// RUN: %clang_cc1 -triple=x86_64-pc-linux-gnu -std=c11 -fsyntax-only -Wno-format -Wformat-signedness -verify=okay %s +// okay-no-diagnostics + +int printf(const char *restrict format, ...); +int scanf(const char * restrict, ...); + +void test_printf_bool(_Bool x) +{ + printf("%d", x); // no-warning + printf("%u", x); // no-warning + printf("%x", x); // no-warning +} + +void test_printf_char(char x) +{ + printf("%c", x); // no-warning +} + +void test_printf_unsigned_char(unsigned char x) +{ + printf("%c", x); // no-warning +} + +void test_printf_int(int x) +{ + printf("%d", x); // no-warning + printf("%u", x); // expected-warning{{format specifies type 'unsigned int' but the argument has type 'int'}} + printf("%x", x); // expected-warning{{format specifies type 'unsigned int' but the argument has type 'int'}} +} + +void test_printf_unsigned(unsigned x) +{ + printf("%d", x); // expected-warning{{format specifies type 'int' but the argument has type 'unsigned int'}} + printf("%u", x); // no-warning + printf("%x", x); // no-warning +} + +void test_printf_long(long x) +{ + printf("%ld", x); // no-warning + printf("%lu", x); // expected-warning{{format specifies type 'unsigned long' but the argument has type 'long'}} + printf("%lx", x); // expected-warning{{format specifies type 'unsigned long' but the argument has type 'long'}} +} + +void test_printf_unsigned_long(unsigned long x) +{ + printf("%ld", x); // expected-warning{{format specifies type 'long' but the argument has type 'unsigned long'}} + printf("%lu", x); // no-warning + printf("%lx", x); // no-warning +} + +void test_printf_long_long(long long x) +{ + printf("%lld", x); // no-warning + printf("%llu", x); // expected-warning{{format specifies type 'unsigned long long' but the argument has type 'long long'}} + printf("%llx", x); // expected-warning{{format specifies type 'unsigned long long' but the argument has type 'long long'}} +} + +void test_printf_unsigned_long_long(unsigned long long x) +{ + printf("%lld", x); // expected-warning{{format specifies type 'long long' but the argument has type 'unsigned long long'}} + printf("%llu", x); // no-warning + printf("%llx", x); // no-warning +} + +enum enum_int { + minus_1 = -1 +}; + +void test_printf_enum_int(enum enum_int x) +{ + printf("%d", x); // no-warning + printf("%u", x); // expected-warning{{format specifies type 'unsigned int' but the argument has underlying type 'int'}} + printf("%x", x); // expected-warning{{format specifies type 'unsigned int' but the argument has underlying type 'int'}} +} + +#ifndef _WIN32 // Disabled due to enums have different underlying type on _WIN32 +enum enum_unsigned { + zero = 0 +}; + +void test_printf_enum_unsigned(enum enum_unsigned x) +{ + printf("%d", x); // expected-warning{{format specifies type 'int' but the argument has underlying type 'unsigned int'}} + printf("%u", x); // no-warning + printf("%x", x); // no-warning +} + +enum enum_long { + minus_one = -1, + int_val = __INT_MAX__, // INT_MAX + unsigned_val = (unsigned)(-__INT_MAX__ -1) // (unsigned)INT_MIN +}; + +void test_printf_enum_long(enum enum_long x) +{ + printf("%ld", x); // no-warning + printf("%lu", x); // expected-warning{{format specifies type 'unsigned long' but the argument has underlying type 'long'}} + printf("%lx", x); // expected-warning{{format specifies type 'unsigned long' but the argument has underlying type 'long'}} +} + +enum enum_unsigned_long { + uint_max_plus = (unsigned long)(__INT_MAX__ *2U +1U)+1, // (unsigned long)UINT_MAX+1 +}; + +void test_printf_enum_unsigned_long(enum enum_unsigned_long x) +{ + printf("%ld", x); // expected-warning{{format specifies type 'long' but the argument has underlying type 'unsigned long'}} + printf("%lu", x); // no-warning + printf("%lx", x); // no-warning +} +#endif + +void test_scanf_char(char *y) { + scanf("%c", y); // no-warning +} + +void test_scanf_unsigned_char(unsigned char *y) { + scanf("%c", y); // no-warning +} + +void test_scanf_int(int *x) { + scanf("%d", x); // no-warning + scanf("%u", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'int *'}} + scanf("%x", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'int *'}} +} + +void test_scanf_unsigned(unsigned *x) { + scanf("%d", x); // expected-warning{{format specifies type 'int *' but the argument has type 'unsigned int *'}} + scanf("%u", x); // no-warning + scanf("%x", x); // no-warning +} + +void test_scanf_long(long *x) { + scanf("%ld", x); // no-warning + scanf("%lu", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'long *'}} + scanf("%lx", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'long *'}} +} + +void test_scanf_unsigned_long(unsigned long *x) { + scanf("%ld", x); // expected-warning{{format specifies type 'long *' but the argument has type 'unsigned long *'}} + scanf("%lu", x); // no-warning + scanf("%lx", x); // no-warning +} + +void test_scanf_longlong(long long *x) { + scanf("%lld", x); // no-warning + scanf("%llu", x); // expected-warning{{format specifies type 'unsigned long long *' but the argument has type 'long long *'}} + scanf("%llx", x); // expected-warning{{format specifies type 'unsigned long long *' but the argument has type 'long long *'}} +} + +void test_scanf_unsigned_longlong(unsigned long long *x) { + scanf("%lld", x); // expected-warning{{format specifies type 'long long *' but the argument has type 'unsigned long long *'}} + scanf("%llu", x); // no-warning + scanf("%llx", x); // no-warning +} + +void test_scanf_enum_int(enum enum_int *x) { + scanf("%d", x); // no-warning + scanf("%u", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'enum enum_int *'}} + scanf("%x", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'enum enum_int *'}} +} + +#ifndef _WIN32 // Disabled due to enums have different underlying type on _WIN32 +void test_scanf_enum_unsigned(enum enum_unsigned *x) { + scanf("%d", x); // expected-warning{{format specifies type 'int *' but the argument has type 'enum enum_unsigned *'}} + scanf("%u", x); // no-warning + scanf("%x", x); // no-warning +} + +void test_scanf_enum_long(enum enum_long *x) { + scanf("%ld", x); // no-warning + scanf("%lu", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'enum enum_long *'}} + scanf("%lx", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'enum enum_long *'}} +} + +void test_scanf_enum_unsigned_long(enum enum_unsigned_long *x) { + scanf("%ld", x); // expected-warning{{format specifies type 'long *' but the argument has type 'enum enum_unsigned_long *'}} + scanf("%lu", x); // no-warning + scanf("%lx", x); // no-warning +} +#endif + +// Verify that we get no warnings from + +typedef short int int16_t; +typedef unsigned short int uint16_t; + +void test_printf_priX16(int16_t x) { + printf("PRId16: %" "d" /*PRId16*/ "\n", x); // no-warning + printf("PRIi16: %" "i" /*PRIi16*/ "\n", x); // no-warning +} + +void test_printf_unsigned_priX16(uint16_t x) { + printf("PRIo16: %" "o" /*PRIo16*/ "\n", x); // no-warning + printf("PRIu16: %" "u" /*PRIu16*/ "\n", x); // no-warning + printf("PRIx16: %" "x" /*PRIx16*/ "\n", x); // no-warning + printf("PRIX16: %" "X" /*PRIX16*/ "\n", x); // no-warning +} + +// Verify that we can suppress a -Wformat-signedness warning by ignoring +// -Wformat (gcc compat). +void test_suppress(int x) +{ +#pragma GCC diagnostic ignored "-Wformat" + printf("%u", x); +} -- GitLab From d12e45ad16a62f7d3ff20b90863f42c9ddb0e624 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Sun, 31 Mar 2024 06:24:19 -0700 Subject: [PATCH 0060/1690] [BOLT][NFC] Split out DomTree construction from BF::calculateLoopInfo (#87181) --- bolt/include/bolt/Core/BinaryFunction.h | 10 ++++++++++ bolt/lib/Core/BinaryFunction.cpp | 12 ++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h index c170fa6397cc..fd3eb2d87128 100644 --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -27,6 +27,7 @@ #include "bolt/Core/BinaryBasicBlock.h" #include "bolt/Core/BinaryContext.h" +#include "bolt/Core/BinaryDomTree.h" #include "bolt/Core/BinaryLoop.h" #include "bolt/Core/BinarySection.h" #include "bolt/Core/DebugData.h" @@ -266,6 +267,7 @@ private: BinaryContext &BC; std::unique_ptr BLI; + std::unique_ptr BDT; /// All labels in the function that are referenced via relocations from /// data objects. Typically these are jump table destinations and computed @@ -838,6 +840,14 @@ public: /// stats. void calculateMacroOpFusionStats(); + /// Returns if BinaryDominatorTree has been constructed for this function. + bool hasDomTree() const { return BDT != nullptr; } + + BinaryDominatorTree &getDomTree() { return *BDT.get(); } + + /// Constructs DomTree for this function. + void constructDomTree(); + /// Returns if loop detection has been run for this function. bool hasLoopInfo() const { return BLI != nullptr; } diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index c9e037c225dd..24f21cb39329 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -12,7 +12,6 @@ #include "bolt/Core/BinaryFunction.h" #include "bolt/Core/BinaryBasicBlock.h" -#include "bolt/Core/BinaryDomTree.h" #include "bolt/Core/DynoStats.h" #include "bolt/Core/HashUtilities.h" #include "bolt/Core/MCPlusBuilder.h" @@ -4076,12 +4075,17 @@ BinaryFunction::~BinaryFunction() { delete BB; } +void BinaryFunction::constructDomTree() { + BDT.reset(new BinaryDominatorTree); + BDT->recalculate(*this); +} + void BinaryFunction::calculateLoopInfo() { + if (!hasDomTree()) + constructDomTree(); // Discover loops. - BinaryDominatorTree DomTree; - DomTree.recalculate(*this); BLI.reset(new BinaryLoopInfo()); - BLI->analyze(DomTree); + BLI->analyze(getDomTree()); // Traverse discovered loops and add depth and profile information. std::stack St; -- GitLab From b6f6be4b500ff64c23a5103ac3311cb74519542f Mon Sep 17 00:00:00 2001 From: Piotr Zegar Date: Sun, 31 Mar 2024 14:34:25 +0000 Subject: [PATCH 0061/1690] [clang-tidy][NFC] Remove duplicated code Remove duplicated matchers by moving some of them to utils/Matchers.h. Add some anonymous namespaces and renamed some code to avoid ODR issues. --- .../clang-tidy/abseil/AbseilMatcher.h | 5 ++ .../abseil/DurationFactoryFloatCheck.cpp | 14 ++-- .../abseil/StringFindStrContainsCheck.cpp | 11 ++-- .../abseil/TimeSubtractionCheck.cpp | 22 +++---- .../UpgradeDurationConversionsCheck.cpp | 4 +- .../bugprone/BadSignalToKillThreadCheck.cpp | 8 +-- .../bugprone/BadSignalToKillThreadCheck.h | 3 + .../DynamicStaticInitializersCheck.cpp | 4 ++ .../EasilySwappableParametersCheck.cpp | 4 ++ .../clang-tidy/bugprone/EmptyCatchCheck.cpp | 5 +- .../bugprone/IncDecInConditionsCheck.cpp | 3 + .../clang-tidy/bugprone/InfiniteLoopCheck.cpp | 8 +-- .../MultipleNewInOneExpressionCheck.cpp | 31 ++------- .../bugprone/MultipleStatementMacroCheck.cpp | 16 +++-- .../bugprone/SignalHandlerCheck.cpp | 4 +- .../bugprone/SizeofExpressionCheck.cpp | 19 +++--- .../bugprone/StringConstructorCheck.cpp | 12 +--- .../bugprone/UnhandledExceptionAtNewCheck.cpp | 29 ++------ .../bugprone/VirtualNearMissCheck.cpp | 7 +- .../cppcoreguidelines/ProTypeVarargCheck.cpp | 4 +- .../VirtualClassDestructorCheck.cpp | 2 + .../google/UpgradeGoogletestCaseCheck.cpp | 2 +- .../clang-tidy/llvmlibc/NamespaceConstants.h | 5 ++ .../modernize/AvoidCArraysCheck.cpp | 10 +-- .../clang-tidy/modernize/LoopConvertCheck.cpp | 4 +- .../clang-tidy/modernize/UseAutoCheck.cpp | 16 ++--- .../clang-tidy/modernize/UseEmplaceCheck.cpp | 8 +-- .../modernize/UseEqualsDefaultCheck.cpp | 4 +- .../modernize/UseEqualsDeleteCheck.cpp | 12 ++-- .../modernize/UseTransparentFunctorsCheck.cpp | 10 +-- .../clang-tidy/objc/MissingHashCheck.cpp | 7 +- .../clang-tidy/objc/ObjcMatcher.h | 40 +++++++++++ .../clang-tidy/objc/SuperSelfCheck.cpp | 22 +------ .../UnnecessaryCopyInitialization.cpp | 2 +- .../ConvertMemberFunctionsToStatic.cpp | 52 +++++---------- .../FunctionCognitiveComplexityCheck.cpp | 23 +++---- .../MakeMemberFunctionConstCheck.cpp | 49 ++++---------- .../readability/RedundantDeclarationCheck.cpp | 3 + .../readability/RedundantSmartptrGetCheck.cpp | 5 +- .../ReferenceToConstructedTemporaryCheck.cpp | 3 +- .../StaticAccessedThroughInstanceCheck.cpp | 7 +- .../clang-tidy/utils/LexerUtils.cpp | 7 ++ .../clang-tidy/utils/LexerUtils.h | 5 ++ clang-tools-extra/clang-tidy/utils/Matchers.h | 66 +++++++++++++++++++ .../clang-tidy/utils/NamespaceAliaser.cpp | 9 ++- 45 files changed, 298 insertions(+), 288 deletions(-) create mode 100644 clang-tools-extra/clang-tidy/objc/ObjcMatcher.h diff --git a/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h b/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h index 1eef86ddc00b..b20d1b1fa7e3 100644 --- a/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h +++ b/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h @@ -6,6 +6,9 @@ // //===----------------------------------------------------------------------===// +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_ABSEILMATCHER_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_ABSEILMATCHER_H + #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" #include @@ -57,3 +60,5 @@ AST_POLYMORPHIC_MATCHER( } } // namespace clang::ast_matchers + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_ABSEILMATCHER_H diff --git a/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp b/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp index 6cb687de4dc8..448bb4b7e1f9 100644 --- a/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp +++ b/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "DurationFactoryFloatCheck.h" +#include "../utils/LexerUtils.h" #include "DurationRewriter.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -18,15 +19,6 @@ using namespace clang::ast_matchers; namespace clang::tidy::abseil { -// Returns `true` if `Range` is inside a macro definition. -static bool insideMacroDefinition(const MatchFinder::MatchResult &Result, - SourceRange Range) { - return !clang::Lexer::makeFileCharRange( - clang::CharSourceRange::getCharRange(Range), - *Result.SourceManager, Result.Context->getLangOpts()) - .isValid(); -} - void DurationFactoryFloatCheck::registerMatchers(MatchFinder *Finder) { Finder->addMatcher( callExpr(callee(functionDecl(DurationFactoryFunction())), @@ -45,7 +37,9 @@ void DurationFactoryFloatCheck::check(const MatchFinder::MatchResult &Result) { const auto *MatchedCall = Result.Nodes.getNodeAs("call"); // Don't try and replace things inside of macro definitions. - if (insideMacroDefinition(Result, MatchedCall->getSourceRange())) + if (tidy::utils::lexer::insideMacroDefinition(MatchedCall->getSourceRange(), + *Result.SourceManager, + Result.Context->getLangOpts())) return; const Expr *Arg = MatchedCall->getArg(0)->IgnoreImpCasts(); diff --git a/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp b/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp index 28b2f81fdc8c..687171e4c782 100644 --- a/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp +++ b/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp @@ -30,12 +30,15 @@ using ::clang::transformer::makeRule; using ::clang::transformer::node; using ::clang::transformer::RewriteRuleWith; +namespace { + AST_MATCHER(Type, isCharType) { return Node.isCharType(); } +} // namespace -static const char DefaultStringLikeClasses[] = "::std::basic_string;" +static const char StringLikeClassesDefault[] = "::std::basic_string;" "::std::basic_string_view;" "::absl::string_view"; -static const char DefaultAbseilStringsMatchHeader[] = "absl/strings/match.h"; +static const char AbseilStringsMatchHeaderDefault[] = "absl/strings/match.h"; static transformer::RewriteRuleWith makeRewriteRule(ArrayRef StringLikeClassNames, @@ -84,9 +87,9 @@ StringFindStrContainsCheck::StringFindStrContainsCheck( StringRef Name, ClangTidyContext *Context) : TransformerClangTidyCheck(Name, Context), StringLikeClassesOption(utils::options::parseStringList( - Options.get("StringLikeClasses", DefaultStringLikeClasses))), + Options.get("StringLikeClasses", StringLikeClassesDefault))), AbseilStringsMatchHeaderOption(Options.get( - "AbseilStringsMatchHeader", DefaultAbseilStringsMatchHeader)) { + "AbseilStringsMatchHeader", AbseilStringsMatchHeaderDefault)) { setRule( makeRewriteRule(StringLikeClassesOption, AbseilStringsMatchHeaderOption)); } diff --git a/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp b/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp index a2171aad32b7..ecf9a2293c9a 100644 --- a/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp +++ b/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "TimeSubtractionCheck.h" +#include "../utils/LexerUtils.h" #include "DurationRewriter.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -18,15 +19,6 @@ using namespace clang::ast_matchers; namespace clang::tidy::abseil { -// Returns `true` if `Range` is inside a macro definition. -static bool insideMacroDefinition(const MatchFinder::MatchResult &Result, - SourceRange Range) { - return !clang::Lexer::makeFileCharRange( - clang::CharSourceRange::getCharRange(Range), - *Result.SourceManager, Result.Context->getLangOpts()) - .isValid(); -} - static bool isConstructorAssignment(const MatchFinder::MatchResult &Result, const Expr *Node) { // For C++14 and earlier there are elidable constructors that must be matched @@ -130,7 +122,9 @@ void TimeSubtractionCheck::check(const MatchFinder::MatchResult &Result) { const auto *BinOp = Result.Nodes.getNodeAs("binop"); std::string InverseName = Result.Nodes.getNodeAs("func_decl")->getNameAsString(); - if (insideMacroDefinition(Result, BinOp->getSourceRange())) + if (tidy::utils::lexer::insideMacroDefinition(BinOp->getSourceRange(), + *Result.SourceManager, + Result.Context->getLangOpts())) return; std::optional Scale = getScaleForTimeInverse(InverseName); @@ -139,7 +133,9 @@ void TimeSubtractionCheck::check(const MatchFinder::MatchResult &Result) { const auto *OuterCall = Result.Nodes.getNodeAs("outer_call"); if (OuterCall) { - if (insideMacroDefinition(Result, OuterCall->getSourceRange())) + if (tidy::utils::lexer::insideMacroDefinition( + OuterCall->getSourceRange(), *Result.SourceManager, + Result.Context->getLangOpts())) return; // We're working with the first case of matcher, and need to replace the @@ -165,7 +161,9 @@ void TimeSubtractionCheck::check(const MatchFinder::MatchResult &Result) { .bind("arg"))), *BinOp, *Result.Context)); if (MaybeCallArg && MaybeCallArg->getArg(0)->IgnoreImpCasts() == BinOp && - !insideMacroDefinition(Result, MaybeCallArg->getSourceRange())) { + !tidy::utils::lexer::insideMacroDefinition( + MaybeCallArg->getSourceRange(), *Result.SourceManager, + Result.Context->getLangOpts())) { // Handle the case where the matched expression is inside a call which // converts it from the inverse to a Duration. In this case, we replace // the outer with just the subtraction expression, which gives the right diff --git a/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp b/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp index 8e27e8e3e0c2..323d966711dd 100644 --- a/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp +++ b/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp @@ -139,8 +139,8 @@ void UpgradeDurationConversionsCheck::check( // We gather source locations from template matches not in template // instantiations for future matches. - internal::Matcher IsInsideTemplate = - hasAncestor(decl(anyOf(classTemplateDecl(), functionTemplateDecl()))); + auto IsInsideTemplate = stmt( + hasAncestor(decl(anyOf(classTemplateDecl(), functionTemplateDecl())))); if (!match(IsInsideTemplate, *ArgExpr, *Result.Context).empty()) MatchedTemplateLocations.insert(Loc); diff --git a/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp index 8c13ce5a90e9..939c311b56b3 100644 --- a/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp @@ -25,15 +25,13 @@ void BadSignalToKillThreadCheck::registerMatchers(MatchFinder *Finder) { this); } -static Preprocessor *PP; - void BadSignalToKillThreadCheck::check(const MatchFinder::MatchResult &Result) { const auto IsSigterm = [](const auto &KeyValue) -> bool { return KeyValue.first->getName() == "SIGTERM" && KeyValue.first->hasMacroDefinition(); }; const auto TryExpandAsInteger = - [](Preprocessor::macro_iterator It) -> std::optional { + [this](Preprocessor::macro_iterator It) -> std::optional { if (It == PP->macro_end()) return std::nullopt; const MacroInfo *MI = PP->getMacroInfo(It->first); @@ -64,8 +62,8 @@ void BadSignalToKillThreadCheck::check(const MatchFinder::MatchResult &Result) { } void BadSignalToKillThreadCheck::registerPPCallbacks( - const SourceManager &SM, Preprocessor *Pp, Preprocessor *ModuleExpanderPP) { - PP = Pp; + const SourceManager &SM, Preprocessor *PP, Preprocessor *ModuleExpanderPP) { + this->PP = PP; } } // namespace clang::tidy::bugprone diff --git a/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.h b/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.h index f21b8c09eb0c..17580e744be9 100644 --- a/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.h +++ b/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.h @@ -26,7 +26,10 @@ public: void check(const ast_matchers::MatchFinder::MatchResult &Result) override; void registerPPCallbacks(const SourceManager &SM, Preprocessor *PP, Preprocessor *ModuleExpanderPP) override; + +private: std::optional SigtermValue; + Preprocessor *PP = nullptr; }; } // namespace clang::tidy::bugprone diff --git a/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp index 93f35cb2c1a3..2c56a39bbeb1 100644 --- a/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp @@ -15,6 +15,8 @@ using namespace clang::ast_matchers; namespace clang::tidy::bugprone { +namespace { + AST_MATCHER(clang::VarDecl, hasConstantDeclaration) { const Expr *Init = Node.getInit(); if (Init && !Init->isValueDependent()) { @@ -25,6 +27,8 @@ AST_MATCHER(clang::VarDecl, hasConstantDeclaration) { return false; } +} // namespace + DynamicStaticInitializersCheck::DynamicStaticInitializersCheck( StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), diff --git a/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp index 84e99c7fafc7..6e7c017aca89 100644 --- a/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp @@ -1881,6 +1881,8 @@ static bool prefixSuffixCoverUnderThreshold(std::size_t Threshold, } // namespace filter +namespace { + /// Matches functions that have at least the specified amount of parameters. AST_MATCHER_P(FunctionDecl, parameterCountGE, unsigned, N) { return Node.getNumParams() >= N; @@ -1903,6 +1905,8 @@ AST_MATCHER(FunctionDecl, isOverloadedUnaryOrBinaryOperator) { } } +} // namespace + /// Returns the DefaultMinimumLength if the Value of requested minimum length /// is less than 2. Minimum lengths of 0 or 1 are not accepted. static inline unsigned clampMinimumLength(const unsigned Value) { diff --git a/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp index ff5b885aa95f..3c64479249a9 100644 --- a/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp @@ -20,7 +20,7 @@ using ::clang::ast_matchers::internal::Matcher; namespace clang::tidy::bugprone { namespace { -AST_MATCHER(CXXCatchStmt, isInMacro) { +AST_MATCHER(CXXCatchStmt, isCatchInMacro) { return Node.getBeginLoc().isMacroID() || Node.getEndLoc().isMacroID() || Node.getCatchLoc().isMacroID(); } @@ -89,7 +89,8 @@ void EmptyCatchCheck::registerMatchers(MatchFinder *Finder) { hasCanonicalType(AllowedNamedExceptionTypes))); Finder->addMatcher( - cxxCatchStmt(unless(isExpansionInSystemHeader()), unless(isInMacro()), + cxxCatchStmt(unless(isExpansionInSystemHeader()), + unless(isCatchInMacro()), unless(hasCaughtType(IgnoredExceptionType)), hasHandler(compoundStmt( statementCountIs(0), diff --git a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp index 9b3b01eb0268..d6645a0e91c1 100644 --- a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp @@ -15,6 +15,7 @@ using namespace clang::ast_matchers; namespace clang::tidy::bugprone { +namespace { AST_MATCHER(BinaryOperator, isLogicalOperator) { return Node.isLogicalOp(); } AST_MATCHER(UnaryOperator, isUnaryPrePostOperator) { @@ -26,6 +27,8 @@ AST_MATCHER(CXXOperatorCallExpr, isPrePostOperator) { Node.getOperator() == OO_MinusMinus; } +} // namespace + void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) { auto OperatorMatcher = expr( anyOf(binaryOperator(anyOf(isComparisonOperator(), isLogicalOperator())), diff --git a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp index e329588290cd..b91c585489c7 100644 --- a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp @@ -33,11 +33,11 @@ AST_MATCHER(FunctionType, typeHasNoReturnAttr) { } // namespace ast_matchers namespace tidy::bugprone { -static internal::Matcher -loopEndingStmt(internal::Matcher Internal) { - internal::Matcher isNoReturnFunType = +static ast_matchers::internal::Matcher +loopEndingStmt(ast_matchers::internal::Matcher Internal) { + ast_matchers::internal::Matcher isNoReturnFunType = ignoringParens(functionType(typeHasNoReturnAttr())); - internal::Matcher isNoReturnDecl = + ast_matchers::internal::Matcher isNoReturnDecl = anyOf(declHasNoReturnAttr(), functionDecl(hasType(isNoReturnFunType)), varDecl(hasType(blockPointerType(pointee(isNoReturnFunType))))); diff --git a/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp index 41191a3cfed2..becfdc5ee6e9 100644 --- a/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "MultipleNewInOneExpressionCheck.h" +#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" #include "clang/Lex/Lexer.h" @@ -51,29 +52,6 @@ bool isExprValueStored(const Expr *E, ASTContext &C) { } // namespace -AST_MATCHER_P(CXXTryStmt, hasHandlerFor, - ast_matchers::internal::Matcher, InnerMatcher) { - for (unsigned NH = Node.getNumHandlers(), I = 0; I < NH; ++I) { - const CXXCatchStmt *CatchS = Node.getHandler(I); - // Check for generic catch handler (match anything). - if (CatchS->getCaughtType().isNull()) - return true; - ast_matchers::internal::BoundNodesTreeBuilder Result(*Builder); - if (InnerMatcher.matches(CatchS->getCaughtType(), Finder, &Result)) { - *Builder = std::move(Result); - return true; - } - } - return false; -} - -AST_MATCHER(CXXNewExpr, mayThrow) { - FunctionDecl *OperatorNew = Node.getOperatorNew(); - if (!OperatorNew) - return false; - return !OperatorNew->getType()->castAs()->isNothrow(); -} - void MultipleNewInOneExpressionCheck::registerMatchers(MatchFinder *Finder) { auto BadAllocType = recordType(hasDeclaration(cxxRecordDecl(hasName("::std::bad_alloc")))); @@ -85,9 +63,10 @@ void MultipleNewInOneExpressionCheck::registerMatchers(MatchFinder *Finder) { auto CatchBadAllocType = qualType(hasCanonicalType(anyOf(BadAllocType, BadAllocReferenceType, ExceptionType, ExceptionReferenceType))); - auto BadAllocCatchingTryBlock = cxxTryStmt(hasHandlerFor(CatchBadAllocType)); + auto BadAllocCatchingTryBlock = + cxxTryStmt(matchers::hasHandlerFor(CatchBadAllocType)); - auto NewExprMayThrow = cxxNewExpr(mayThrow()); + auto NewExprMayThrow = cxxNewExpr(matchers::mayThrow()); auto HasNewExpr1 = expr(anyOf(NewExprMayThrow.bind("new1"), hasDescendant(NewExprMayThrow.bind("new1")))); auto HasNewExpr2 = expr(anyOf(NewExprMayThrow.bind("new2"), @@ -115,7 +94,7 @@ void MultipleNewInOneExpressionCheck::registerMatchers(MatchFinder *Finder) { hasAncestor(BadAllocCatchingTryBlock)), this); Finder->addMatcher( - cxxNewExpr(mayThrow(), + cxxNewExpr(matchers::mayThrow(), hasDescendant(NewExprMayThrow.bind("new2_in_new1")), hasAncestor(BadAllocCatchingTryBlock)) .bind("new1"), diff --git a/clang-tools-extra/clang-tidy/bugprone/MultipleStatementMacroCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MultipleStatementMacroCheck.cpp index de05cc0e4f7f..5746a1f3b358 100644 --- a/clang-tools-extra/clang-tidy/bugprone/MultipleStatementMacroCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/MultipleStatementMacroCheck.cpp @@ -16,10 +16,13 @@ namespace clang::tidy::bugprone { namespace { -AST_MATCHER(Expr, isInMacro) { return Node.getBeginLoc().isMacroID(); } +AST_MATCHER(Expr, isExprInMacro) { return Node.getBeginLoc().isMacroID(); } + +} // namespace /// Find the next statement after `S`. -const Stmt *nextStmt(const MatchFinder::MatchResult &Result, const Stmt *S) { +static const Stmt *nextStmt(const MatchFinder::MatchResult &Result, + const Stmt *S) { auto Parents = Result.Context->getParents(*S); if (Parents.empty()) return nullptr; @@ -40,8 +43,8 @@ using ExpansionRanges = std::vector; /// \brief Get all the macro expansion ranges related to `Loc`. /// /// The result is ordered from most inner to most outer. -ExpansionRanges getExpansionRanges(SourceLocation Loc, - const MatchFinder::MatchResult &Result) { +static ExpansionRanges +getExpansionRanges(SourceLocation Loc, const MatchFinder::MatchResult &Result) { ExpansionRanges Locs; while (Loc.isMacroID()) { Locs.push_back( @@ -51,10 +54,9 @@ ExpansionRanges getExpansionRanges(SourceLocation Loc, return Locs; } -} // namespace - void MultipleStatementMacroCheck::registerMatchers(MatchFinder *Finder) { - const auto Inner = expr(isInMacro(), unless(compoundStmt())).bind("inner"); + const auto Inner = + expr(isExprInMacro(), unless(compoundStmt())).bind("inner"); Finder->addMatcher( stmt(anyOf(ifStmt(hasThen(Inner)), ifStmt(hasElse(Inner)).bind("else"), whileStmt(hasBody(Inner)), forStmt(hasBody(Inner)))) diff --git a/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp index 902490f4d33c..93412c213bbd 100644 --- a/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp @@ -322,12 +322,12 @@ SourceRange getSourceRangeOfStmt(const Stmt *S, ASTContext &Ctx) { return P.getSourceRange(); } -} // namespace - AST_MATCHER(FunctionDecl, isStandardFunction) { return isStandardFunction(&Node); } +} // namespace + SignalHandlerCheck::SignalHandlerCheck(StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), diff --git a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp index a1cffbc66619..77cc9f39d302 100644 --- a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp @@ -17,10 +17,6 @@ namespace clang::tidy::bugprone { namespace { -AST_MATCHER_P(IntegerLiteral, isBiggerThan, unsigned, N) { - return Node.getValue().ugt(N); -} - AST_MATCHER_P2(Expr, hasSizeOfDescendant, int, Depth, ast_matchers::internal::Matcher, InnerMatcher) { if (Depth < 0) @@ -48,15 +44,15 @@ AST_MATCHER_P2(Expr, hasSizeOfDescendant, int, Depth, return false; } -CharUnits getSizeOfType(const ASTContext &Ctx, const Type *Ty) { +} // namespace + +static CharUnits getSizeOfType(const ASTContext &Ctx, const Type *Ty) { if (!Ty || Ty->isIncompleteType() || Ty->isDependentType() || isa(Ty) || !Ty->isConstantSizeType()) return CharUnits::Zero(); return Ctx.getTypeSizeInChars(Ty); } -} // namespace - SizeofExpressionCheck::SizeofExpressionCheck(StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), @@ -183,10 +179,11 @@ void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) { // Detect expression like: sizeof(expr) <= k for a suspicious constant 'k'. if (WarnOnSizeOfCompareToConstant) { Finder->addMatcher( - binaryOperator(matchers::isRelationalOperator(), - hasOperands(ignoringParenImpCasts(SizeOfExpr), - ignoringParenImpCasts(integerLiteral(anyOf( - equals(0), isBiggerThan(0x80000)))))) + binaryOperator( + matchers::isRelationalOperator(), + hasOperands(ignoringParenImpCasts(SizeOfExpr), + ignoringParenImpCasts(integerLiteral(anyOf( + equals(0), matchers::isBiggerThan(0x80000)))))) .bind("sizeof-compare-constant"), this); } diff --git a/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp index 8ae4351ac283..11d105e23fca 100644 --- a/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "StringConstructorCheck.h" +#include "../utils/Matchers.h" #include "../utils/OptionsUtils.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -16,12 +17,7 @@ using namespace clang::ast_matchers; namespace clang::tidy::bugprone { -namespace { -AST_MATCHER_P(IntegerLiteral, isBiggerThan, unsigned, N) { - return Node.getValue().getZExtValue() > N; -} - -const char DefaultStringNames[] = +static const char DefaultStringNames[] = "::std::basic_string;::std::basic_string_view"; static std::vector @@ -36,8 +32,6 @@ removeNamespaces(const std::vector &Names) { return Result; } -} // namespace - StringConstructorCheck::StringConstructorCheck(StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), @@ -61,7 +55,7 @@ void StringConstructorCheck::registerMatchers(MatchFinder *Finder) { unaryOperator(hasOperatorName("-"), hasUnaryOperand(integerLiteral(unless(equals(0))))))); const auto LargeLengthExpr = expr(ignoringParenImpCasts( - integerLiteral(isBiggerThan(LargeLengthThreshold)))); + integerLiteral(matchers::isBiggerThan(LargeLengthThreshold)))); const auto CharPtrType = type(anyOf(pointerType(), arrayType())); // Match a string-literal; even through a declaration with initializer. diff --git a/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp index b160e7259905..155dced4a136 100644 --- a/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "UnhandledExceptionAtNewCheck.h" +#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -14,29 +15,6 @@ using namespace clang::ast_matchers; namespace clang::tidy::bugprone { -AST_MATCHER_P(CXXTryStmt, hasHandlerFor, - ast_matchers::internal::Matcher, InnerMatcher) { - for (unsigned NH = Node.getNumHandlers(), I = 0; I < NH; ++I) { - const CXXCatchStmt *CatchS = Node.getHandler(I); - // Check for generic catch handler (match anything). - if (CatchS->getCaughtType().isNull()) - return true; - ast_matchers::internal::BoundNodesTreeBuilder Result(*Builder); - if (InnerMatcher.matches(CatchS->getCaughtType(), Finder, &Result)) { - *Builder = std::move(Result); - return true; - } - } - return false; -} - -AST_MATCHER(CXXNewExpr, mayThrow) { - FunctionDecl *OperatorNew = Node.getOperatorNew(); - if (!OperatorNew) - return false; - return !OperatorNew->getType()->castAs()->isNothrow(); -} - UnhandledExceptionAtNewCheck::UnhandledExceptionAtNewCheck( StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context) {} @@ -52,11 +30,12 @@ void UnhandledExceptionAtNewCheck::registerMatchers(MatchFinder *Finder) { auto CatchBadAllocType = qualType(hasCanonicalType(anyOf(BadAllocType, BadAllocReferenceType, ExceptionType, ExceptionReferenceType))); - auto BadAllocCatchingTryBlock = cxxTryStmt(hasHandlerFor(CatchBadAllocType)); + auto BadAllocCatchingTryBlock = + cxxTryStmt(matchers::hasHandlerFor(CatchBadAllocType)); auto FunctionMayNotThrow = functionDecl(isNoThrow()); - Finder->addMatcher(cxxNewExpr(mayThrow(), + Finder->addMatcher(cxxNewExpr(matchers::mayThrow(), unless(hasAncestor(BadAllocCatchingTryBlock)), hasAncestor(FunctionMayNotThrow)) .bind("new-expr"), diff --git a/clang-tools-extra/clang-tidy/bugprone/VirtualNearMissCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/VirtualNearMissCheck.cpp index bebdce525e28..bd73a7cac378 100644 --- a/clang-tools-extra/clang-tidy/bugprone/VirtualNearMissCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/VirtualNearMissCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "VirtualNearMissCheck.h" +#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/AST/CXXInheritance.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -17,8 +18,6 @@ using namespace clang::ast_matchers; namespace clang::tidy::bugprone { namespace { -AST_MATCHER(CXXMethodDecl, isStatic) { return Node.isStatic(); } - AST_MATCHER(CXXMethodDecl, isOverloadedOperator) { return Node.isOverloadedOperator(); } @@ -216,8 +215,8 @@ void VirtualNearMissCheck::registerMatchers(MatchFinder *Finder) { Finder->addMatcher( cxxMethodDecl( unless(anyOf(isOverride(), isImplicit(), cxxConstructorDecl(), - cxxDestructorDecl(), cxxConversionDecl(), isStatic(), - isOverloadedOperator()))) + cxxDestructorDecl(), cxxConversionDecl(), + matchers::isStaticMethod(), isOverloadedOperator()))) .bind("method"), this); } diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.cpp index 3923df312791..d933c043da64 100644 --- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.cpp +++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.cpp @@ -20,7 +20,9 @@ using namespace clang::ast_matchers; namespace clang::tidy::cppcoreguidelines { -const internal::VariadicDynCastAllOfMatcher VAArgExpr; +const clang::ast_matchers::internal::VariadicDynCastAllOfMatcher + VAArgExpr; static constexpr StringRef AllowedVariadics[] = { // clang-format off diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp index aa70b3896f16..a93b603099b8 100644 --- a/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp +++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp @@ -18,6 +18,7 @@ using namespace clang::ast_matchers; namespace clang::tidy::cppcoreguidelines { +namespace { AST_MATCHER(CXXRecordDecl, hasPublicVirtualOrProtectedNonVirtualDestructor) { // We need to call Node.getDestructor() instead of matching a // CXXDestructorDecl. Otherwise, tests will fail for class templates, since @@ -33,6 +34,7 @@ AST_MATCHER(CXXRecordDecl, hasPublicVirtualOrProtectedNonVirtualDestructor) { !Destructor->isVirtual())); } +} // namespace void VirtualClassDestructorCheck::registerMatchers(MatchFinder *Finder) { ast_matchers::internal::Matcher InheritsVirtualMethod = hasAnyBase(hasType(cxxRecordDecl(has(cxxMethodDecl(isVirtual()))))); diff --git a/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp b/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp index 805dcaf3ce40..c7590b515232 100644 --- a/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp +++ b/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp @@ -234,7 +234,7 @@ static bool isInInstantiation(const NodeType &Node, template static bool isInTemplate(const NodeType &Node, const MatchFinder::MatchResult &Result) { - internal::Matcher IsInsideTemplate = + ast_matchers::internal::Matcher IsInsideTemplate = hasAncestor(decl(anyOf(classTemplateDecl(), functionTemplateDecl()))); return !match(IsInsideTemplate, Node, *Result.Context).empty(); } diff --git a/clang-tools-extra/clang-tidy/llvmlibc/NamespaceConstants.h b/clang-tools-extra/clang-tidy/llvmlibc/NamespaceConstants.h index 7d4120085b86..77d4bd0a2541 100644 --- a/clang-tools-extra/clang-tidy/llvmlibc/NamespaceConstants.h +++ b/clang-tools-extra/clang-tidy/llvmlibc/NamespaceConstants.h @@ -6,6 +6,9 @@ // //===----------------------------------------------------------------------===// +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_LLVMLIBC_NAMESPACE_CONSTANTS_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_LLVMLIBC_NAMESPACE_CONSTANTS_H + #include "llvm/ADT/StringRef.h" namespace clang::tidy::llvm_libc { @@ -14,3 +17,5 @@ const static llvm::StringRef RequiredNamespaceStart = "__llvm_libc"; const static llvm::StringRef RequiredNamespaceMacroName = "LIBC_NAMESPACE"; } // namespace clang::tidy::llvm_libc + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_LLVMLIBC_NAMESPACE_CONSTANTS_H diff --git a/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp b/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp index 89790ea70cf2..517288930fa8 100644 --- a/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp @@ -20,14 +20,6 @@ AST_MATCHER(clang::TypeLoc, hasValidBeginLoc) { return Node.getBeginLoc().isValid(); } -AST_MATCHER_P(clang::TypeLoc, hasType, - clang::ast_matchers::internal::Matcher, - InnerMatcher) { - const clang::Type *TypeNode = Node.getTypePtr(); - return TypeNode != nullptr && - InnerMatcher.matches(*TypeNode, Finder, Builder); -} - AST_MATCHER(clang::RecordDecl, isExternCContext) { return Node.isExternCContext(); } @@ -59,7 +51,7 @@ void AvoidCArraysCheck::registerMatchers(MatchFinder *Finder) { unless(parmVarDecl()))))); Finder->addMatcher( - typeLoc(hasValidBeginLoc(), hasType(arrayType()), + typeLoc(hasValidBeginLoc(), loc(arrayType()), unless(anyOf(hasParent(parmVarDecl(isArgvOfMain())), hasParent(varDecl(isExternC())), hasParent(fieldDecl( diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp index 3229e302eb43..9b05c3a7425c 100644 --- a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp @@ -94,7 +94,7 @@ static StatementMatcher incrementVarMatcher() { } static StatementMatcher -arrayConditionMatcher(internal::Matcher LimitExpr) { +arrayConditionMatcher(ast_matchers::internal::Matcher LimitExpr) { return binaryOperator( anyOf(allOf(hasOperatorName("<"), hasLHS(integerComparisonMatcher()), hasRHS(LimitExpr)), @@ -209,7 +209,7 @@ StatementMatcher makeIteratorLoopMatcher(bool IsReverse) { // This matcher tests that a declaration is a CXXRecordDecl that has an // overloaded operator*(). If the operator*() returns by value instead of by // reference then the return type is tagged with DerefByValueResultName. - internal::Matcher TestDerefReturnsByValue = + ast_matchers::internal::Matcher TestDerefReturnsByValue = hasType(hasUnqualifiedDesugaredType( recordType(hasDeclaration(cxxRecordDecl(hasMethod(cxxMethodDecl( hasOverloadedOperatorName("*"), diff --git a/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp index aec67808846b..dc8a87b8ae0f 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "UseAutoCheck.h" +#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/AST/TypeLoc.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -20,6 +21,7 @@ using namespace clang::ast_matchers; using namespace clang::ast_matchers::internal; namespace clang::tidy::modernize { + namespace { const char IteratorDeclStmtId[] = "iterator_decl"; @@ -155,14 +157,6 @@ Matcher hasStdContainerName() { return hasAnyName(ContainerNames); } -/// Matches declaration reference or member expressions with explicit template -/// arguments. -AST_POLYMORPHIC_MATCHER(hasExplicitTemplateArgs, - AST_POLYMORPHIC_SUPPORTED_TYPES(DeclRefExpr, - MemberExpr)) { - return Node.hasExplicitTemplateArgs(); -} - /// Returns a DeclarationMatcher that matches standard iterators nested /// inside records with a standard container name. DeclarationMatcher standardIterator() { @@ -238,9 +232,9 @@ StatementMatcher makeDeclWithTemplateCastMatcher() { auto ST = substTemplateTypeParmType(hasReplacementType(equalsBoundNode("arg"))); - auto ExplicitCall = - anyOf(has(memberExpr(hasExplicitTemplateArgs())), - has(ignoringImpCasts(declRefExpr(hasExplicitTemplateArgs())))); + auto ExplicitCall = anyOf( + has(memberExpr(matchers::hasExplicitTemplateArgs())), + has(ignoringImpCasts(declRefExpr(matchers::hasExplicitTemplateArgs())))); auto TemplateArg = hasTemplateArgument(0, refersToType(qualType().bind("arg"))); diff --git a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp index 430455a38f39..c477e566c787 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp @@ -7,7 +7,9 @@ //===----------------------------------------------------------------------===// #include "UseEmplaceCheck.h" +#include "../utils/Matchers.h" #include "../utils/OptionsUtils.h" + using namespace clang::ast_matchers; namespace clang::tidy::modernize { @@ -78,10 +80,6 @@ AST_MATCHER(CXXMemberCallExpr, hasSameNumArgsAsDeclNumParams) { return Node.getNumArgs() == Node.getMethodDecl()->getNumParams(); } -AST_MATCHER(DeclRefExpr, hasExplicitTemplateArgs) { - return Node.hasExplicitTemplateArgs(); -} - // Helper Matcher which applies the given QualType Matcher either directly or by // resolving a pointer type to its pointee. Used to match v.push_back() as well // as p->push_back(). @@ -221,7 +219,7 @@ void UseEmplaceCheck::registerMatchers(MatchFinder *Finder) { auto MakeTuple = ignoringImplicit( callExpr(callee(expr(ignoringImplicit(declRefExpr( - unless(hasExplicitTemplateArgs()), + unless(matchers::hasExplicitTemplateArgs()), to(functionDecl(hasAnyName(TupleMakeFunctions)))))))) .bind("make")); diff --git a/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp index 5134eb51a032..406be8c66be7 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp @@ -48,8 +48,8 @@ static std::set getAllDirectBases(const CXXRecordDecl *Record) { /// Returns a matcher that matches member expressions where the base is /// the variable declared as \p Var and the accessed member is the one declared /// as \p Field. -internal::Matcher accessToFieldInVar(const FieldDecl *Field, - const ValueDecl *Var) { +ast_matchers::internal::Matcher accessToFieldInVar(const FieldDecl *Field, + const ValueDecl *Var) { return ignoringImpCasts( memberExpr(hasObjectExpression(declRefExpr(to(varDecl(equalsNode(Var))))), member(fieldDecl(equalsNode(Field))))); diff --git a/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp index 9561cc71183d..9fdd60ead040 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp @@ -16,6 +16,7 @@ using namespace clang::ast_matchers; namespace clang::tidy::modernize { namespace { + AST_MATCHER(FunctionDecl, hasAnyDefinition) { if (Node.hasBody() || Node.isPureVirtual() || Node.isDefaulted() || Node.isDeleted()) @@ -41,9 +42,6 @@ AST_MATCHER(CXXMethodDecl, isSpecialFunction) { } } // namespace -static const char SpecialFunction[] = "SpecialFunction"; -static const char DeletedNotPublic[] = "DeletedNotPublic"; - UseEqualsDeleteCheck::UseEqualsDeleteCheck(StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), @@ -63,17 +61,17 @@ void UseEqualsDeleteCheck::registerMatchers(MatchFinder *Finder) { // defined. unless(ofClass(hasMethod(cxxMethodDecl(unless(PrivateSpecialFn), unless(hasAnyDefinition())))))) - .bind(SpecialFunction), + .bind("SpecialFunction"), this); Finder->addMatcher( - cxxMethodDecl(isDeleted(), unless(isPublic())).bind(DeletedNotPublic), + cxxMethodDecl(isDeleted(), unless(isPublic())).bind("DeletedNotPublic"), this); } void UseEqualsDeleteCheck::check(const MatchFinder::MatchResult &Result) { if (const auto *Func = - Result.Nodes.getNodeAs(SpecialFunction)) { + Result.Nodes.getNodeAs("SpecialFunction")) { SourceLocation EndLoc = Lexer::getLocForEndOfToken( Func->getEndLoc(), 0, *Result.SourceManager, getLangOpts()); @@ -84,7 +82,7 @@ void UseEqualsDeleteCheck::check(const MatchFinder::MatchResult &Result) { "use '= delete' to prohibit calling of a special member function") << FixItHint::CreateInsertion(EndLoc, " = delete"); } else if (const auto *Func = - Result.Nodes.getNodeAs(DeletedNotPublic)) { + Result.Nodes.getNodeAs("DeletedNotPublic")) { // Ignore this warning in macros, since it's extremely noisy in code using // DISALLOW_COPY_AND_ASSIGN-style macros and there's no easy way to // automatically fix the warning when macros are in play. diff --git a/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp index 2223a1999f73..225a1e7bedea 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp @@ -62,7 +62,7 @@ void UseTransparentFunctorsCheck::registerMatchers(MatchFinder *Finder) { this); } -static const StringRef Message = "prefer transparent functors '%0<>'"; +static const StringRef MessageDiag = "prefer transparent functors '%0<>'"; template static T getInnerTypeLocAs(TypeLoc Loc) { T Result; @@ -79,7 +79,7 @@ void UseTransparentFunctorsCheck::check( Result.Nodes.getNodeAs("FunctorClass"); if (const auto *FuncInst = Result.Nodes.getNodeAs("FuncInst")) { - diag(FuncInst->getBeginLoc(), Message) << FuncClass->getName(); + diag(FuncInst->getBeginLoc(), MessageDiag) << FuncClass->getName(); return; } @@ -117,9 +117,9 @@ void UseTransparentFunctorsCheck::check( SourceLocation ReportLoc = FunctorLoc.getLocation(); if (ReportLoc.isInvalid()) return; - diag(ReportLoc, Message) << FuncClass->getName() - << FixItHint::CreateRemoval( - FunctorTypeLoc.getArgLoc(0).getSourceRange()); + diag(ReportLoc, MessageDiag) + << FuncClass->getName() + << FixItHint::CreateRemoval(FunctorTypeLoc.getArgLoc(0).getSourceRange()); } } // namespace clang::tidy::modernize diff --git a/clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp b/clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp index 42f383edc67e..103268caf32c 100644 --- a/clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp +++ b/clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "MissingHashCheck.h" +#include "ObjcMatcher.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -16,12 +17,6 @@ namespace clang::tidy::objc { namespace { -AST_MATCHER_P(ObjCImplementationDecl, hasInterface, - ast_matchers::internal::Matcher, Base) { - const ObjCInterfaceDecl *InterfaceDecl = Node.getClassInterface(); - return Base.matches(*InterfaceDecl, Finder, Builder); -} - AST_MATCHER_P(ObjCContainerDecl, hasInstanceMethod, ast_matchers::internal::Matcher, Base) { // Check each instance method against the provided matcher. diff --git a/clang-tools-extra/clang-tidy/objc/ObjcMatcher.h b/clang-tools-extra/clang-tidy/objc/ObjcMatcher.h new file mode 100644 index 000000000000..f272b5b0f8a6 --- /dev/null +++ b/clang-tools-extra/clang-tidy/objc/ObjcMatcher.h @@ -0,0 +1,40 @@ +//===- AbseilMatcher.h - clang-tidy ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_OBJCMATCHER_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_OBJCMATCHER_H + +#include "clang/AST/ASTContext.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" + +namespace clang::ast_matchers { + +/// Matches Objective-C implementations with interfaces that match +/// \c Base. +/// +/// Example matches implementation declarations for X. +/// (matcher = objcImplementationDecl(hasInterface(hasName("X")))) +/// \code +/// @interface X +/// @end +/// @implementation X +/// @end +/// @interface Y +// @end +/// @implementation Y +/// @end +/// \endcode +AST_MATCHER_P(ObjCImplementationDecl, hasInterface, + ast_matchers::internal::Matcher, Base) { + const ObjCInterfaceDecl *InterfaceDecl = Node.getClassInterface(); + return Base.matches(*InterfaceDecl, Finder, Builder); +} + +} // namespace clang::ast_matchers + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_OBJCMATCHER_H diff --git a/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.cpp b/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.cpp index 951cbc52c9a9..572a1463e5c0 100644 --- a/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.cpp +++ b/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "SuperSelfCheck.h" +#include "ObjcMatcher.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -32,27 +33,6 @@ AST_MATCHER(ObjCMethodDecl, isInitializer) { return Node.getMethodFamily() == OMF_init; } -/// Matches Objective-C implementations with interfaces that match -/// \c Base. -/// -/// Example matches implementation declarations for X. -/// (matcher = objcImplementationDecl(hasInterface(hasName("X")))) -/// \code -/// @interface X -/// @end -/// @implementation X -/// @end -/// @interface Y -// @end -/// @implementation Y -/// @end -/// \endcode -AST_MATCHER_P(ObjCImplementationDecl, hasInterface, - ast_matchers::internal::Matcher, Base) { - const ObjCInterfaceDecl *InterfaceDecl = Node.getClassInterface(); - return Base.matches(*InterfaceDecl, Finder, Builder); -} - /// Matches Objective-C message expressions where the receiver is the /// super instance. /// diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp index 9beb185cba92..8c38bad593a3 100644 --- a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp +++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp @@ -232,7 +232,7 @@ UnnecessaryCopyInitialization::UnnecessaryCopyInitialization( Options.get("ExcludedContainerTypes", ""))) {} void UnnecessaryCopyInitialization::registerMatchers(MatchFinder *Finder) { - auto LocalVarCopiedFrom = [this](const internal::Matcher &CopyCtorArg) { + auto LocalVarCopiedFrom = [this](const auto &CopyCtorArg) { return compoundStmt( forEachDescendant( declStmt( diff --git a/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp b/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp index 1284df6bd99c..5b1bf0108cbb 100644 --- a/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp +++ b/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "ConvertMemberFunctionsToStatic.h" +#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/RecursiveASTVisitor.h" @@ -18,40 +19,12 @@ using namespace clang::ast_matchers; namespace clang::tidy::readability { -AST_MATCHER(CXXMethodDecl, isStatic) { return Node.isStatic(); } - -AST_MATCHER(CXXMethodDecl, hasTrivialBody) { return Node.hasTrivialBody(); } +namespace { AST_MATCHER(CXXMethodDecl, isOverloadedOperator) { return Node.isOverloadedOperator(); } -AST_MATCHER(CXXRecordDecl, hasAnyDependentBases) { - return Node.hasAnyDependentBases(); -} - -AST_MATCHER(CXXMethodDecl, isTemplate) { - return Node.getTemplatedKind() != FunctionDecl::TK_NonTemplate; -} - -AST_MATCHER(CXXMethodDecl, isDependentContext) { - return Node.isDependentContext(); -} - -AST_MATCHER(CXXMethodDecl, isInsideMacroDefinition) { - const ASTContext &Ctxt = Finder->getASTContext(); - return clang::Lexer::makeFileCharRange( - clang::CharSourceRange::getCharRange( - Node.getTypeSourceInfo()->getTypeLoc().getSourceRange()), - Ctxt.getSourceManager(), Ctxt.getLangOpts()) - .isInvalid(); -} - -AST_MATCHER_P(CXXMethodDecl, hasCanonicalDecl, - ast_matchers::internal::Matcher, InnerMatcher) { - return InnerMatcher.matches(*Node.getCanonicalDecl(), Finder, Builder); -} - AST_MATCHER(CXXMethodDecl, usesThis) { class FindUsageOfThis : public RecursiveASTVisitor { public: @@ -74,22 +47,27 @@ AST_MATCHER(CXXMethodDecl, usesThis) { return UsageOfThis.Used; } +} // namespace + void ConvertMemberFunctionsToStatic::registerMatchers(MatchFinder *Finder) { Finder->addMatcher( cxxMethodDecl( isDefinition(), isUserProvided(), unless(anyOf( - isExpansionInSystemHeader(), isVirtual(), isStatic(), - hasTrivialBody(), isOverloadedOperator(), cxxConstructorDecl(), - cxxDestructorDecl(), cxxConversionDecl(), isTemplate(), - isDependentContext(), + isExpansionInSystemHeader(), isVirtual(), + matchers::isStaticMethod(), matchers::hasTrivialBody(), + isOverloadedOperator(), cxxConstructorDecl(), cxxDestructorDecl(), + cxxConversionDecl(), matchers::isTemplate(), + matchers::isDependentContext(), ofClass(anyOf( isLambda(), - hasAnyDependentBases()) // Method might become virtual - // depending on template base class. + matchers::hasAnyDependentBases()) // Method might become + // virtual depending on + // template base class. ), - isInsideMacroDefinition(), - hasCanonicalDecl(isInsideMacroDefinition()), usesThis()))) + matchers::isInsideMacroDefinition(), + matchers::hasCanonicalDecl(matchers::isInsideMacroDefinition()), + usesThis()))) .bind("x"), this); } diff --git a/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp b/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp index 759cdd44fd65..261c02ea2d53 100644 --- a/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp @@ -202,9 +202,9 @@ void CognitiveComplexity::account(SourceLocation Loc, unsigned short Nesting, Total += Increase; } -class FunctionASTVisitor final - : public RecursiveASTVisitor { - using Base = RecursiveASTVisitor; +class CognitiveComplexityFunctionASTVisitor final + : public RecursiveASTVisitor { + using Base = RecursiveASTVisitor; // If set to true, macros are ignored during analysis. const bool IgnoreMacros; @@ -219,7 +219,7 @@ class FunctionASTVisitor final std::stack> BinaryOperatorsStack; public: - explicit FunctionASTVisitor(const bool IgnoreMacros) + explicit CognitiveComplexityFunctionASTVisitor(const bool IgnoreMacros) : IgnoreMacros(IgnoreMacros) {} bool traverseStmtWithIncreasedNestingLevel(Stmt *Node) { @@ -453,12 +453,13 @@ public: // The parameter MainAnalyzedFunction is needed to differentiate between the // cases where TraverseDecl() is the entry point from // FunctionCognitiveComplexityCheck::check() and the cases where it was called - // from the FunctionASTVisitor itself. Explanation: if we get a function - // definition (e.g. constructor, destructor, method), the Cognitive Complexity - // specification states that the Nesting level shall be increased. But if this - // function is the entry point, then the Nesting level should not be - // increased. Thus that parameter is there and is used to fall-through - // directly to traversing if this is the main function that is being analyzed. + // from the CognitiveComplexityFunctionASTVisitor itself. Explanation: if we + // get a function definition (e.g. constructor, destructor, method), the + // Cognitive Complexity specification states that the Nesting level shall be + // increased. But if this function is the entry point, then the Nesting level + // should not be increased. Thus that parameter is there and is used to + // fall-through directly to traversing if this is the main function that is + // being analyzed. bool TraverseDecl(Decl *Node, bool MainAnalyzedFunction = false) { if (!Node || MainAnalyzedFunction) return Base::TraverseDecl(Node); @@ -515,7 +516,7 @@ void FunctionCognitiveComplexityCheck::registerMatchers(MatchFinder *Finder) { void FunctionCognitiveComplexityCheck::check( const MatchFinder::MatchResult &Result) { - FunctionASTVisitor Visitor(IgnoreMacros); + CognitiveComplexityFunctionASTVisitor Visitor(IgnoreMacros); SourceLocation Loc; const auto *TheDecl = Result.Nodes.getNodeAs("func"); diff --git a/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp b/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp index d42fcba70e81..c828ef3bc78b 100644 --- a/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "MakeMemberFunctionConstCheck.h" +#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/AST/ParentMapContext.h" #include "clang/AST/RecursiveASTVisitor.h" @@ -17,36 +18,6 @@ using namespace clang::ast_matchers; namespace clang::tidy::readability { -AST_MATCHER(CXXMethodDecl, isStatic) { return Node.isStatic(); } - -AST_MATCHER(CXXMethodDecl, hasTrivialBody) { return Node.hasTrivialBody(); } - -AST_MATCHER(CXXRecordDecl, hasAnyDependentBases) { - return Node.hasAnyDependentBases(); -} - -AST_MATCHER(CXXMethodDecl, isTemplate) { - return Node.getTemplatedKind() != FunctionDecl::TK_NonTemplate; -} - -AST_MATCHER(CXXMethodDecl, isDependentContext) { - return Node.isDependentContext(); -} - -AST_MATCHER(CXXMethodDecl, isInsideMacroDefinition) { - const ASTContext &Ctxt = Finder->getASTContext(); - return clang::Lexer::makeFileCharRange( - clang::CharSourceRange::getCharRange( - Node.getTypeSourceInfo()->getTypeLoc().getSourceRange()), - Ctxt.getSourceManager(), Ctxt.getLangOpts()) - .isInvalid(); -} - -AST_MATCHER_P(CXXMethodDecl, hasCanonicalDecl, - ast_matchers::internal::Matcher, InnerMatcher) { - return InnerMatcher.matches(*Node.getCanonicalDecl(), Finder, Builder); -} - enum UsageKind { Unused, Const, NonConst }; class FindUsageOfThis : public RecursiveASTVisitor { @@ -205,6 +176,7 @@ public: } }; +namespace { AST_MATCHER(CXXMethodDecl, usesThisAsConst) { FindUsageOfThis UsageOfThis(Finder->getASTContext()); @@ -214,6 +186,8 @@ AST_MATCHER(CXXMethodDecl, usesThisAsConst) { return UsageOfThis.Usage == Const; } +} // namespace + void MakeMemberFunctionConstCheck::registerMatchers(MatchFinder *Finder) { Finder->addMatcher( traverse( @@ -222,15 +196,18 @@ void MakeMemberFunctionConstCheck::registerMatchers(MatchFinder *Finder) { isDefinition(), isUserProvided(), unless(anyOf( isExpansionInSystemHeader(), isVirtual(), isConst(), - isStatic(), hasTrivialBody(), cxxConstructorDecl(), - cxxDestructorDecl(), isTemplate(), isDependentContext(), - ofClass(anyOf(isLambda(), - hasAnyDependentBases()) // Method might become + matchers::isStaticMethod(), matchers::hasTrivialBody(), + cxxConstructorDecl(), cxxDestructorDecl(), + matchers::isTemplate(), matchers::isDependentContext(), + ofClass(anyOf( + isLambda(), + matchers::hasAnyDependentBases()) // Method might become // virtual depending on // template base class. ), - isInsideMacroDefinition(), - hasCanonicalDecl(isInsideMacroDefinition()))), + matchers::isInsideMacroDefinition(), + matchers::hasCanonicalDecl( + matchers::isInsideMacroDefinition()))), usesThisAsConst()) .bind("x")), this); diff --git a/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp index 7850a6f29995..bfd4dca11e94 100644 --- a/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp @@ -15,10 +15,13 @@ using namespace clang::ast_matchers; namespace clang::tidy::readability { +namespace { AST_MATCHER(FunctionDecl, doesDeclarationForceExternallyVisibleDefinition) { return Node.doesDeclarationForceExternallyVisibleDefinition(); } +} // namespace + RedundantDeclarationCheck::RedundantDeclarationCheck(StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), diff --git a/clang-tools-extra/clang-tidy/readability/RedundantSmartptrGetCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantSmartptrGetCheck.cpp index 8837ac16e882..86d3c7e860c9 100644 --- a/clang-tools-extra/clang-tidy/readability/RedundantSmartptrGetCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/RedundantSmartptrGetCheck.cpp @@ -15,7 +15,8 @@ using namespace clang::ast_matchers; namespace clang::tidy::readability { namespace { -internal::Matcher callToGet(const internal::Matcher &OnClass) { +clang::ast_matchers::internal::Matcher +callToGet(const clang::ast_matchers::internal::Matcher &OnClass) { return expr( anyOf(cxxMemberCallExpr( on(expr(anyOf(hasType(OnClass), @@ -43,7 +44,7 @@ internal::Matcher callToGet(const internal::Matcher &OnClass) { .bind("redundant_get"); } -internal::Matcher knownSmartptr() { +clang::ast_matchers::internal::Matcher knownSmartptr() { return recordDecl(hasAnyName("::std::unique_ptr", "::std::shared_ptr")); } diff --git a/clang-tools-extra/clang-tidy/readability/ReferenceToConstructedTemporaryCheck.cpp b/clang-tools-extra/clang-tidy/readability/ReferenceToConstructedTemporaryCheck.cpp index 587ae8ea3058..9abe0f0e155a 100644 --- a/clang-tools-extra/clang-tidy/readability/ReferenceToConstructedTemporaryCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/ReferenceToConstructedTemporaryCheck.cpp @@ -20,7 +20,8 @@ namespace { // Predicate structure to check if lifetime of temporary is not extended by // ValueDecl pointed out by ID struct NotExtendedByDeclBoundToPredicate { - bool operator()(const internal::BoundNodesMap &Nodes) const { + bool + operator()(const clang::ast_matchers::internal::BoundNodesMap &Nodes) const { const auto *Other = Nodes.getNodeAs(ID); if (!Other) return true; diff --git a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp index 65356cc3929c..77b8f5e5ac21 100644 --- a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "StaticAccessedThroughInstanceCheck.h" +#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" #include "llvm/ADT/StringRef.h" @@ -15,10 +16,6 @@ using namespace clang::ast_matchers; namespace clang::tidy::readability { -namespace { -AST_MATCHER(CXXMethodDecl, isStatic) { return Node.isStatic(); } -} // namespace - static unsigned getNameSpecifierNestingLevel(const QualType &QType) { if (const auto *ElType = QType->getAs()) { if (const NestedNameSpecifier *NestedSpecifiers = ElType->getQualifier()) { @@ -42,7 +39,7 @@ void StaticAccessedThroughInstanceCheck::storeOptions( void StaticAccessedThroughInstanceCheck::registerMatchers(MatchFinder *Finder) { Finder->addMatcher( - memberExpr(hasDeclaration(anyOf(cxxMethodDecl(isStatic()), + memberExpr(hasDeclaration(anyOf(cxxMethodDecl(matchers::isStaticMethod()), varDecl(hasStaticStorageDuration()), enumConstantDecl()))) .bind("memberExpression"), diff --git a/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp b/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp index df2b0bef576c..8ee148a5bb39 100644 --- a/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp +++ b/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp @@ -285,4 +285,11 @@ SourceLocation getLocationForNoexceptSpecifier(const FunctionDecl *FuncDecl, return {}; } +bool insideMacroDefinition(SourceRange Range, const SourceManager &SM, + const LangOptions &LangOpts) { + return clang::Lexer::makeFileCharRange( + clang::CharSourceRange::getCharRange(Range), SM, LangOpts) + .isInvalid(); +} + } // namespace clang::tidy::utils::lexer diff --git a/clang-tools-extra/clang-tidy/utils/LexerUtils.h b/clang-tools-extra/clang-tidy/utils/LexerUtils.h index ea9bd512b68b..a697e7ee8adf 100644 --- a/clang-tools-extra/clang-tidy/utils/LexerUtils.h +++ b/clang-tools-extra/clang-tidy/utils/LexerUtils.h @@ -125,6 +125,11 @@ SourceLocation getUnifiedEndLoc(const Stmt &S, const SourceManager &SM, SourceLocation getLocationForNoexceptSpecifier(const FunctionDecl *FuncDecl, const SourceManager &SM); +// Returns true if a part of the range resides inside a macro expansion or the +// range does not reside on the same FileID. +bool insideMacroDefinition(SourceRange Range, const SourceManager &SM, + const LangOptions &LangOpts); + } // namespace tidy::utils::lexer } // namespace clang diff --git a/clang-tools-extra/clang-tidy/utils/Matchers.h b/clang-tools-extra/clang-tidy/utils/Matchers.h index 045e3ffbb6a8..dc47bb515e0e 100644 --- a/clang-tools-extra/clang-tidy/utils/Matchers.h +++ b/clang-tools-extra/clang-tidy/utils/Matchers.h @@ -9,6 +9,7 @@ #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_MATCHERS_H #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_MATCHERS_H +#include "LexerUtils.h" #include "TypeTraits.h" #include "clang/AST/ExprConcepts.h" #include "clang/ASTMatchers/ASTMatchers.h" @@ -22,6 +23,10 @@ AST_MATCHER(BinaryOperator, isRelationalOperator) { AST_MATCHER(BinaryOperator, isEqualityOperator) { return Node.isEqualityOp(); } +AST_MATCHER_P(IntegerLiteral, isBiggerThan, unsigned, N) { + return Node.getValue().getZExtValue() > N; +} + AST_MATCHER(QualType, isExpensiveToCopy) { std::optional IsExpensive = utils::type_traits::isExpensiveToCopy(Node, Finder->getASTContext()); @@ -37,6 +42,67 @@ AST_MATCHER(QualType, isTriviallyDestructible) { return utils::type_traits::isTriviallyDestructible(Node); } +AST_MATCHER(CXXRecordDecl, hasAnyDependentBases) { + return Node.hasAnyDependentBases(); +} + +AST_MATCHER(CXXMethodDecl, isStaticMethod) { return Node.isStatic(); } + +AST_MATCHER(CXXMethodDecl, hasTrivialBody) { return Node.hasTrivialBody(); } + +AST_MATCHER(CXXMethodDecl, isTemplate) { + return Node.getTemplatedKind() != FunctionDecl::TK_NonTemplate; +} + +AST_MATCHER(CXXMethodDecl, isDependentContext) { + return Node.isDependentContext(); +} + +AST_MATCHER(CXXMethodDecl, isInsideMacroDefinition) { + const ASTContext &Ctxt = Finder->getASTContext(); + return utils::lexer::insideMacroDefinition( + Node.getTypeSourceInfo()->getTypeLoc().getSourceRange(), + Ctxt.getSourceManager(), Ctxt.getLangOpts()); +} + +AST_MATCHER_P(CXXMethodDecl, hasCanonicalDecl, + ast_matchers::internal::Matcher, InnerMatcher) { + return InnerMatcher.matches(*Node.getCanonicalDecl(), Finder, Builder); +} + +AST_MATCHER(CXXNewExpr, mayThrow) { + FunctionDecl *OperatorNew = Node.getOperatorNew(); + if (!OperatorNew) + return false; + if (auto *FuncType = OperatorNew->getType()->getAs()) + return !FuncType->isNothrow(); + return false; +} + +AST_MATCHER_P(CXXTryStmt, hasHandlerFor, + ast_matchers::internal::Matcher, InnerMatcher) { + for (unsigned NH = Node.getNumHandlers(), I = 0; I < NH; ++I) { + const CXXCatchStmt *CatchS = Node.getHandler(I); + // Check for generic catch handler (match anything). + if (CatchS->getCaughtType().isNull()) + return true; + ast_matchers::internal::BoundNodesTreeBuilder Result(*Builder); + if (InnerMatcher.matches(CatchS->getCaughtType(), Finder, &Result)) { + *Builder = std::move(Result); + return true; + } + } + return false; +} + +// Matches declaration reference or member expressions with explicit template +// arguments. +AST_POLYMORPHIC_MATCHER(hasExplicitTemplateArgs, + AST_POLYMORPHIC_SUPPORTED_TYPES(DeclRefExpr, + MemberExpr)) { + return Node.hasExplicitTemplateArgs(); +} + // Returns QualType matcher for references to const. AST_MATCHER_FUNCTION(ast_matchers::TypeMatcher, isReferenceToConst) { using namespace ast_matchers; diff --git a/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp b/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp index 4703ce126981..214f0fd43a62 100644 --- a/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp +++ b/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp @@ -13,18 +13,23 @@ #include "clang/ASTMatchers/ASTMatchers.h" #include "clang/Lex/Lexer.h" #include + namespace clang::tidy::utils { using namespace ast_matchers; -NamespaceAliaser::NamespaceAliaser(const SourceManager &SourceMgr) - : SourceMgr(SourceMgr) {} +namespace { AST_MATCHER_P(NamespaceAliasDecl, hasTargetNamespace, ast_matchers::internal::Matcher, innerMatcher) { return innerMatcher.matches(*Node.getNamespace(), Finder, Builder); } +} // namespace + +NamespaceAliaser::NamespaceAliaser(const SourceManager &SourceMgr) + : SourceMgr(SourceMgr) {} + std::optional NamespaceAliaser::createAlias(ASTContext &Context, const Stmt &Statement, StringRef Namespace, -- GitLab From 11a411a49b62c129bba551df4587dd446fcdc660 Mon Sep 17 00:00:00 2001 From: Piotr Zegar Date: Sun, 31 Mar 2024 15:06:49 +0000 Subject: [PATCH 0062/1690] Revert "[clang-tidy][NFC] Remove duplicated code" This reverts commit b6f6be4b500ff64c23a5103ac3311cb74519542f. --- .../clang-tidy/abseil/AbseilMatcher.h | 5 -- .../abseil/DurationFactoryFloatCheck.cpp | 14 ++-- .../abseil/StringFindStrContainsCheck.cpp | 11 ++-- .../abseil/TimeSubtractionCheck.cpp | 22 ++++--- .../UpgradeDurationConversionsCheck.cpp | 4 +- .../bugprone/BadSignalToKillThreadCheck.cpp | 8 ++- .../bugprone/BadSignalToKillThreadCheck.h | 3 - .../DynamicStaticInitializersCheck.cpp | 4 -- .../EasilySwappableParametersCheck.cpp | 4 -- .../clang-tidy/bugprone/EmptyCatchCheck.cpp | 5 +- .../bugprone/IncDecInConditionsCheck.cpp | 3 - .../clang-tidy/bugprone/InfiniteLoopCheck.cpp | 8 +-- .../MultipleNewInOneExpressionCheck.cpp | 31 +++++++-- .../bugprone/MultipleStatementMacroCheck.cpp | 16 ++--- .../bugprone/SignalHandlerCheck.cpp | 4 +- .../bugprone/SizeofExpressionCheck.cpp | 19 +++--- .../bugprone/StringConstructorCheck.cpp | 12 +++- .../bugprone/UnhandledExceptionAtNewCheck.cpp | 29 ++++++-- .../bugprone/VirtualNearMissCheck.cpp | 7 +- .../cppcoreguidelines/ProTypeVarargCheck.cpp | 4 +- .../VirtualClassDestructorCheck.cpp | 2 - .../google/UpgradeGoogletestCaseCheck.cpp | 2 +- .../clang-tidy/llvmlibc/NamespaceConstants.h | 5 -- .../modernize/AvoidCArraysCheck.cpp | 10 ++- .../clang-tidy/modernize/LoopConvertCheck.cpp | 4 +- .../clang-tidy/modernize/UseAutoCheck.cpp | 16 +++-- .../clang-tidy/modernize/UseEmplaceCheck.cpp | 8 ++- .../modernize/UseEqualsDefaultCheck.cpp | 4 +- .../modernize/UseEqualsDeleteCheck.cpp | 12 ++-- .../modernize/UseTransparentFunctorsCheck.cpp | 10 +-- .../clang-tidy/objc/MissingHashCheck.cpp | 7 +- .../clang-tidy/objc/ObjcMatcher.h | 40 ----------- .../clang-tidy/objc/SuperSelfCheck.cpp | 22 ++++++- .../UnnecessaryCopyInitialization.cpp | 2 +- .../ConvertMemberFunctionsToStatic.cpp | 52 ++++++++++----- .../FunctionCognitiveComplexityCheck.cpp | 23 ++++--- .../MakeMemberFunctionConstCheck.cpp | 49 ++++++++++---- .../readability/RedundantDeclarationCheck.cpp | 3 - .../readability/RedundantSmartptrGetCheck.cpp | 5 +- .../ReferenceToConstructedTemporaryCheck.cpp | 3 +- .../StaticAccessedThroughInstanceCheck.cpp | 7 +- .../clang-tidy/utils/LexerUtils.cpp | 7 -- .../clang-tidy/utils/LexerUtils.h | 5 -- clang-tools-extra/clang-tidy/utils/Matchers.h | 66 ------------------- .../clang-tidy/utils/NamespaceAliaser.cpp | 9 +-- 45 files changed, 288 insertions(+), 298 deletions(-) delete mode 100644 clang-tools-extra/clang-tidy/objc/ObjcMatcher.h diff --git a/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h b/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h index b20d1b1fa7e3..1eef86ddc00b 100644 --- a/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h +++ b/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h @@ -6,9 +6,6 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_ABSEILMATCHER_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_ABSEILMATCHER_H - #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" #include @@ -60,5 +57,3 @@ AST_POLYMORPHIC_MATCHER( } } // namespace clang::ast_matchers - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_ABSEILMATCHER_H diff --git a/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp b/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp index 448bb4b7e1f9..6cb687de4dc8 100644 --- a/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp +++ b/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "DurationFactoryFloatCheck.h" -#include "../utils/LexerUtils.h" #include "DurationRewriter.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -19,6 +18,15 @@ using namespace clang::ast_matchers; namespace clang::tidy::abseil { +// Returns `true` if `Range` is inside a macro definition. +static bool insideMacroDefinition(const MatchFinder::MatchResult &Result, + SourceRange Range) { + return !clang::Lexer::makeFileCharRange( + clang::CharSourceRange::getCharRange(Range), + *Result.SourceManager, Result.Context->getLangOpts()) + .isValid(); +} + void DurationFactoryFloatCheck::registerMatchers(MatchFinder *Finder) { Finder->addMatcher( callExpr(callee(functionDecl(DurationFactoryFunction())), @@ -37,9 +45,7 @@ void DurationFactoryFloatCheck::check(const MatchFinder::MatchResult &Result) { const auto *MatchedCall = Result.Nodes.getNodeAs("call"); // Don't try and replace things inside of macro definitions. - if (tidy::utils::lexer::insideMacroDefinition(MatchedCall->getSourceRange(), - *Result.SourceManager, - Result.Context->getLangOpts())) + if (insideMacroDefinition(Result, MatchedCall->getSourceRange())) return; const Expr *Arg = MatchedCall->getArg(0)->IgnoreImpCasts(); diff --git a/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp b/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp index 687171e4c782..28b2f81fdc8c 100644 --- a/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp +++ b/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp @@ -30,15 +30,12 @@ using ::clang::transformer::makeRule; using ::clang::transformer::node; using ::clang::transformer::RewriteRuleWith; -namespace { - AST_MATCHER(Type, isCharType) { return Node.isCharType(); } -} // namespace -static const char StringLikeClassesDefault[] = "::std::basic_string;" +static const char DefaultStringLikeClasses[] = "::std::basic_string;" "::std::basic_string_view;" "::absl::string_view"; -static const char AbseilStringsMatchHeaderDefault[] = "absl/strings/match.h"; +static const char DefaultAbseilStringsMatchHeader[] = "absl/strings/match.h"; static transformer::RewriteRuleWith makeRewriteRule(ArrayRef StringLikeClassNames, @@ -87,9 +84,9 @@ StringFindStrContainsCheck::StringFindStrContainsCheck( StringRef Name, ClangTidyContext *Context) : TransformerClangTidyCheck(Name, Context), StringLikeClassesOption(utils::options::parseStringList( - Options.get("StringLikeClasses", StringLikeClassesDefault))), + Options.get("StringLikeClasses", DefaultStringLikeClasses))), AbseilStringsMatchHeaderOption(Options.get( - "AbseilStringsMatchHeader", AbseilStringsMatchHeaderDefault)) { + "AbseilStringsMatchHeader", DefaultAbseilStringsMatchHeader)) { setRule( makeRewriteRule(StringLikeClassesOption, AbseilStringsMatchHeaderOption)); } diff --git a/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp b/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp index ecf9a2293c9a..a2171aad32b7 100644 --- a/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp +++ b/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "TimeSubtractionCheck.h" -#include "../utils/LexerUtils.h" #include "DurationRewriter.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -19,6 +18,15 @@ using namespace clang::ast_matchers; namespace clang::tidy::abseil { +// Returns `true` if `Range` is inside a macro definition. +static bool insideMacroDefinition(const MatchFinder::MatchResult &Result, + SourceRange Range) { + return !clang::Lexer::makeFileCharRange( + clang::CharSourceRange::getCharRange(Range), + *Result.SourceManager, Result.Context->getLangOpts()) + .isValid(); +} + static bool isConstructorAssignment(const MatchFinder::MatchResult &Result, const Expr *Node) { // For C++14 and earlier there are elidable constructors that must be matched @@ -122,9 +130,7 @@ void TimeSubtractionCheck::check(const MatchFinder::MatchResult &Result) { const auto *BinOp = Result.Nodes.getNodeAs("binop"); std::string InverseName = Result.Nodes.getNodeAs("func_decl")->getNameAsString(); - if (tidy::utils::lexer::insideMacroDefinition(BinOp->getSourceRange(), - *Result.SourceManager, - Result.Context->getLangOpts())) + if (insideMacroDefinition(Result, BinOp->getSourceRange())) return; std::optional Scale = getScaleForTimeInverse(InverseName); @@ -133,9 +139,7 @@ void TimeSubtractionCheck::check(const MatchFinder::MatchResult &Result) { const auto *OuterCall = Result.Nodes.getNodeAs("outer_call"); if (OuterCall) { - if (tidy::utils::lexer::insideMacroDefinition( - OuterCall->getSourceRange(), *Result.SourceManager, - Result.Context->getLangOpts())) + if (insideMacroDefinition(Result, OuterCall->getSourceRange())) return; // We're working with the first case of matcher, and need to replace the @@ -161,9 +165,7 @@ void TimeSubtractionCheck::check(const MatchFinder::MatchResult &Result) { .bind("arg"))), *BinOp, *Result.Context)); if (MaybeCallArg && MaybeCallArg->getArg(0)->IgnoreImpCasts() == BinOp && - !tidy::utils::lexer::insideMacroDefinition( - MaybeCallArg->getSourceRange(), *Result.SourceManager, - Result.Context->getLangOpts())) { + !insideMacroDefinition(Result, MaybeCallArg->getSourceRange())) { // Handle the case where the matched expression is inside a call which // converts it from the inverse to a Duration. In this case, we replace // the outer with just the subtraction expression, which gives the right diff --git a/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp b/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp index 323d966711dd..8e27e8e3e0c2 100644 --- a/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp +++ b/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp @@ -139,8 +139,8 @@ void UpgradeDurationConversionsCheck::check( // We gather source locations from template matches not in template // instantiations for future matches. - auto IsInsideTemplate = stmt( - hasAncestor(decl(anyOf(classTemplateDecl(), functionTemplateDecl())))); + internal::Matcher IsInsideTemplate = + hasAncestor(decl(anyOf(classTemplateDecl(), functionTemplateDecl()))); if (!match(IsInsideTemplate, *ArgExpr, *Result.Context).empty()) MatchedTemplateLocations.insert(Loc); diff --git a/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp index 939c311b56b3..8c13ce5a90e9 100644 --- a/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp @@ -25,13 +25,15 @@ void BadSignalToKillThreadCheck::registerMatchers(MatchFinder *Finder) { this); } +static Preprocessor *PP; + void BadSignalToKillThreadCheck::check(const MatchFinder::MatchResult &Result) { const auto IsSigterm = [](const auto &KeyValue) -> bool { return KeyValue.first->getName() == "SIGTERM" && KeyValue.first->hasMacroDefinition(); }; const auto TryExpandAsInteger = - [this](Preprocessor::macro_iterator It) -> std::optional { + [](Preprocessor::macro_iterator It) -> std::optional { if (It == PP->macro_end()) return std::nullopt; const MacroInfo *MI = PP->getMacroInfo(It->first); @@ -62,8 +64,8 @@ void BadSignalToKillThreadCheck::check(const MatchFinder::MatchResult &Result) { } void BadSignalToKillThreadCheck::registerPPCallbacks( - const SourceManager &SM, Preprocessor *PP, Preprocessor *ModuleExpanderPP) { - this->PP = PP; + const SourceManager &SM, Preprocessor *Pp, Preprocessor *ModuleExpanderPP) { + PP = Pp; } } // namespace clang::tidy::bugprone diff --git a/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.h b/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.h index 17580e744be9..f21b8c09eb0c 100644 --- a/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.h +++ b/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.h @@ -26,10 +26,7 @@ public: void check(const ast_matchers::MatchFinder::MatchResult &Result) override; void registerPPCallbacks(const SourceManager &SM, Preprocessor *PP, Preprocessor *ModuleExpanderPP) override; - -private: std::optional SigtermValue; - Preprocessor *PP = nullptr; }; } // namespace clang::tidy::bugprone diff --git a/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp index 2c56a39bbeb1..93f35cb2c1a3 100644 --- a/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp @@ -15,8 +15,6 @@ using namespace clang::ast_matchers; namespace clang::tidy::bugprone { -namespace { - AST_MATCHER(clang::VarDecl, hasConstantDeclaration) { const Expr *Init = Node.getInit(); if (Init && !Init->isValueDependent()) { @@ -27,8 +25,6 @@ AST_MATCHER(clang::VarDecl, hasConstantDeclaration) { return false; } -} // namespace - DynamicStaticInitializersCheck::DynamicStaticInitializersCheck( StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), diff --git a/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp index 6e7c017aca89..84e99c7fafc7 100644 --- a/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp @@ -1881,8 +1881,6 @@ static bool prefixSuffixCoverUnderThreshold(std::size_t Threshold, } // namespace filter -namespace { - /// Matches functions that have at least the specified amount of parameters. AST_MATCHER_P(FunctionDecl, parameterCountGE, unsigned, N) { return Node.getNumParams() >= N; @@ -1905,8 +1903,6 @@ AST_MATCHER(FunctionDecl, isOverloadedUnaryOrBinaryOperator) { } } -} // namespace - /// Returns the DefaultMinimumLength if the Value of requested minimum length /// is less than 2. Minimum lengths of 0 or 1 are not accepted. static inline unsigned clampMinimumLength(const unsigned Value) { diff --git a/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp index 3c64479249a9..ff5b885aa95f 100644 --- a/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp @@ -20,7 +20,7 @@ using ::clang::ast_matchers::internal::Matcher; namespace clang::tidy::bugprone { namespace { -AST_MATCHER(CXXCatchStmt, isCatchInMacro) { +AST_MATCHER(CXXCatchStmt, isInMacro) { return Node.getBeginLoc().isMacroID() || Node.getEndLoc().isMacroID() || Node.getCatchLoc().isMacroID(); } @@ -89,8 +89,7 @@ void EmptyCatchCheck::registerMatchers(MatchFinder *Finder) { hasCanonicalType(AllowedNamedExceptionTypes))); Finder->addMatcher( - cxxCatchStmt(unless(isExpansionInSystemHeader()), - unless(isCatchInMacro()), + cxxCatchStmt(unless(isExpansionInSystemHeader()), unless(isInMacro()), unless(hasCaughtType(IgnoredExceptionType)), hasHandler(compoundStmt( statementCountIs(0), diff --git a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp index d6645a0e91c1..9b3b01eb0268 100644 --- a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp @@ -15,7 +15,6 @@ using namespace clang::ast_matchers; namespace clang::tidy::bugprone { -namespace { AST_MATCHER(BinaryOperator, isLogicalOperator) { return Node.isLogicalOp(); } AST_MATCHER(UnaryOperator, isUnaryPrePostOperator) { @@ -27,8 +26,6 @@ AST_MATCHER(CXXOperatorCallExpr, isPrePostOperator) { Node.getOperator() == OO_MinusMinus; } -} // namespace - void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) { auto OperatorMatcher = expr( anyOf(binaryOperator(anyOf(isComparisonOperator(), isLogicalOperator())), diff --git a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp index b91c585489c7..e329588290cd 100644 --- a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp @@ -33,11 +33,11 @@ AST_MATCHER(FunctionType, typeHasNoReturnAttr) { } // namespace ast_matchers namespace tidy::bugprone { -static ast_matchers::internal::Matcher -loopEndingStmt(ast_matchers::internal::Matcher Internal) { - ast_matchers::internal::Matcher isNoReturnFunType = +static internal::Matcher +loopEndingStmt(internal::Matcher Internal) { + internal::Matcher isNoReturnFunType = ignoringParens(functionType(typeHasNoReturnAttr())); - ast_matchers::internal::Matcher isNoReturnDecl = + internal::Matcher isNoReturnDecl = anyOf(declHasNoReturnAttr(), functionDecl(hasType(isNoReturnFunType)), varDecl(hasType(blockPointerType(pointee(isNoReturnFunType))))); diff --git a/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp index becfdc5ee6e9..41191a3cfed2 100644 --- a/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "MultipleNewInOneExpressionCheck.h" -#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" #include "clang/Lex/Lexer.h" @@ -52,6 +51,29 @@ bool isExprValueStored(const Expr *E, ASTContext &C) { } // namespace +AST_MATCHER_P(CXXTryStmt, hasHandlerFor, + ast_matchers::internal::Matcher, InnerMatcher) { + for (unsigned NH = Node.getNumHandlers(), I = 0; I < NH; ++I) { + const CXXCatchStmt *CatchS = Node.getHandler(I); + // Check for generic catch handler (match anything). + if (CatchS->getCaughtType().isNull()) + return true; + ast_matchers::internal::BoundNodesTreeBuilder Result(*Builder); + if (InnerMatcher.matches(CatchS->getCaughtType(), Finder, &Result)) { + *Builder = std::move(Result); + return true; + } + } + return false; +} + +AST_MATCHER(CXXNewExpr, mayThrow) { + FunctionDecl *OperatorNew = Node.getOperatorNew(); + if (!OperatorNew) + return false; + return !OperatorNew->getType()->castAs()->isNothrow(); +} + void MultipleNewInOneExpressionCheck::registerMatchers(MatchFinder *Finder) { auto BadAllocType = recordType(hasDeclaration(cxxRecordDecl(hasName("::std::bad_alloc")))); @@ -63,10 +85,9 @@ void MultipleNewInOneExpressionCheck::registerMatchers(MatchFinder *Finder) { auto CatchBadAllocType = qualType(hasCanonicalType(anyOf(BadAllocType, BadAllocReferenceType, ExceptionType, ExceptionReferenceType))); - auto BadAllocCatchingTryBlock = - cxxTryStmt(matchers::hasHandlerFor(CatchBadAllocType)); + auto BadAllocCatchingTryBlock = cxxTryStmt(hasHandlerFor(CatchBadAllocType)); - auto NewExprMayThrow = cxxNewExpr(matchers::mayThrow()); + auto NewExprMayThrow = cxxNewExpr(mayThrow()); auto HasNewExpr1 = expr(anyOf(NewExprMayThrow.bind("new1"), hasDescendant(NewExprMayThrow.bind("new1")))); auto HasNewExpr2 = expr(anyOf(NewExprMayThrow.bind("new2"), @@ -94,7 +115,7 @@ void MultipleNewInOneExpressionCheck::registerMatchers(MatchFinder *Finder) { hasAncestor(BadAllocCatchingTryBlock)), this); Finder->addMatcher( - cxxNewExpr(matchers::mayThrow(), + cxxNewExpr(mayThrow(), hasDescendant(NewExprMayThrow.bind("new2_in_new1")), hasAncestor(BadAllocCatchingTryBlock)) .bind("new1"), diff --git a/clang-tools-extra/clang-tidy/bugprone/MultipleStatementMacroCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MultipleStatementMacroCheck.cpp index 5746a1f3b358..de05cc0e4f7f 100644 --- a/clang-tools-extra/clang-tidy/bugprone/MultipleStatementMacroCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/MultipleStatementMacroCheck.cpp @@ -16,13 +16,10 @@ namespace clang::tidy::bugprone { namespace { -AST_MATCHER(Expr, isExprInMacro) { return Node.getBeginLoc().isMacroID(); } - -} // namespace +AST_MATCHER(Expr, isInMacro) { return Node.getBeginLoc().isMacroID(); } /// Find the next statement after `S`. -static const Stmt *nextStmt(const MatchFinder::MatchResult &Result, - const Stmt *S) { +const Stmt *nextStmt(const MatchFinder::MatchResult &Result, const Stmt *S) { auto Parents = Result.Context->getParents(*S); if (Parents.empty()) return nullptr; @@ -43,8 +40,8 @@ using ExpansionRanges = std::vector; /// \brief Get all the macro expansion ranges related to `Loc`. /// /// The result is ordered from most inner to most outer. -static ExpansionRanges -getExpansionRanges(SourceLocation Loc, const MatchFinder::MatchResult &Result) { +ExpansionRanges getExpansionRanges(SourceLocation Loc, + const MatchFinder::MatchResult &Result) { ExpansionRanges Locs; while (Loc.isMacroID()) { Locs.push_back( @@ -54,9 +51,10 @@ getExpansionRanges(SourceLocation Loc, const MatchFinder::MatchResult &Result) { return Locs; } +} // namespace + void MultipleStatementMacroCheck::registerMatchers(MatchFinder *Finder) { - const auto Inner = - expr(isExprInMacro(), unless(compoundStmt())).bind("inner"); + const auto Inner = expr(isInMacro(), unless(compoundStmt())).bind("inner"); Finder->addMatcher( stmt(anyOf(ifStmt(hasThen(Inner)), ifStmt(hasElse(Inner)).bind("else"), whileStmt(hasBody(Inner)), forStmt(hasBody(Inner)))) diff --git a/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp index 93412c213bbd..902490f4d33c 100644 --- a/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp @@ -322,12 +322,12 @@ SourceRange getSourceRangeOfStmt(const Stmt *S, ASTContext &Ctx) { return P.getSourceRange(); } +} // namespace + AST_MATCHER(FunctionDecl, isStandardFunction) { return isStandardFunction(&Node); } -} // namespace - SignalHandlerCheck::SignalHandlerCheck(StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), diff --git a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp index 77cc9f39d302..a1cffbc66619 100644 --- a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp @@ -17,6 +17,10 @@ namespace clang::tidy::bugprone { namespace { +AST_MATCHER_P(IntegerLiteral, isBiggerThan, unsigned, N) { + return Node.getValue().ugt(N); +} + AST_MATCHER_P2(Expr, hasSizeOfDescendant, int, Depth, ast_matchers::internal::Matcher, InnerMatcher) { if (Depth < 0) @@ -44,15 +48,15 @@ AST_MATCHER_P2(Expr, hasSizeOfDescendant, int, Depth, return false; } -} // namespace - -static CharUnits getSizeOfType(const ASTContext &Ctx, const Type *Ty) { +CharUnits getSizeOfType(const ASTContext &Ctx, const Type *Ty) { if (!Ty || Ty->isIncompleteType() || Ty->isDependentType() || isa(Ty) || !Ty->isConstantSizeType()) return CharUnits::Zero(); return Ctx.getTypeSizeInChars(Ty); } +} // namespace + SizeofExpressionCheck::SizeofExpressionCheck(StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), @@ -179,11 +183,10 @@ void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) { // Detect expression like: sizeof(expr) <= k for a suspicious constant 'k'. if (WarnOnSizeOfCompareToConstant) { Finder->addMatcher( - binaryOperator( - matchers::isRelationalOperator(), - hasOperands(ignoringParenImpCasts(SizeOfExpr), - ignoringParenImpCasts(integerLiteral(anyOf( - equals(0), matchers::isBiggerThan(0x80000)))))) + binaryOperator(matchers::isRelationalOperator(), + hasOperands(ignoringParenImpCasts(SizeOfExpr), + ignoringParenImpCasts(integerLiteral(anyOf( + equals(0), isBiggerThan(0x80000)))))) .bind("sizeof-compare-constant"), this); } diff --git a/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp index 11d105e23fca..8ae4351ac283 100644 --- a/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/StringConstructorCheck.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "StringConstructorCheck.h" -#include "../utils/Matchers.h" #include "../utils/OptionsUtils.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -17,7 +16,12 @@ using namespace clang::ast_matchers; namespace clang::tidy::bugprone { -static const char DefaultStringNames[] = +namespace { +AST_MATCHER_P(IntegerLiteral, isBiggerThan, unsigned, N) { + return Node.getValue().getZExtValue() > N; +} + +const char DefaultStringNames[] = "::std::basic_string;::std::basic_string_view"; static std::vector @@ -32,6 +36,8 @@ removeNamespaces(const std::vector &Names) { return Result; } +} // namespace + StringConstructorCheck::StringConstructorCheck(StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), @@ -55,7 +61,7 @@ void StringConstructorCheck::registerMatchers(MatchFinder *Finder) { unaryOperator(hasOperatorName("-"), hasUnaryOperand(integerLiteral(unless(equals(0))))))); const auto LargeLengthExpr = expr(ignoringParenImpCasts( - integerLiteral(matchers::isBiggerThan(LargeLengthThreshold)))); + integerLiteral(isBiggerThan(LargeLengthThreshold)))); const auto CharPtrType = type(anyOf(pointerType(), arrayType())); // Match a string-literal; even through a declaration with initializer. diff --git a/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp index 155dced4a136..b160e7259905 100644 --- a/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "UnhandledExceptionAtNewCheck.h" -#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -15,6 +14,29 @@ using namespace clang::ast_matchers; namespace clang::tidy::bugprone { +AST_MATCHER_P(CXXTryStmt, hasHandlerFor, + ast_matchers::internal::Matcher, InnerMatcher) { + for (unsigned NH = Node.getNumHandlers(), I = 0; I < NH; ++I) { + const CXXCatchStmt *CatchS = Node.getHandler(I); + // Check for generic catch handler (match anything). + if (CatchS->getCaughtType().isNull()) + return true; + ast_matchers::internal::BoundNodesTreeBuilder Result(*Builder); + if (InnerMatcher.matches(CatchS->getCaughtType(), Finder, &Result)) { + *Builder = std::move(Result); + return true; + } + } + return false; +} + +AST_MATCHER(CXXNewExpr, mayThrow) { + FunctionDecl *OperatorNew = Node.getOperatorNew(); + if (!OperatorNew) + return false; + return !OperatorNew->getType()->castAs()->isNothrow(); +} + UnhandledExceptionAtNewCheck::UnhandledExceptionAtNewCheck( StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context) {} @@ -30,12 +52,11 @@ void UnhandledExceptionAtNewCheck::registerMatchers(MatchFinder *Finder) { auto CatchBadAllocType = qualType(hasCanonicalType(anyOf(BadAllocType, BadAllocReferenceType, ExceptionType, ExceptionReferenceType))); - auto BadAllocCatchingTryBlock = - cxxTryStmt(matchers::hasHandlerFor(CatchBadAllocType)); + auto BadAllocCatchingTryBlock = cxxTryStmt(hasHandlerFor(CatchBadAllocType)); auto FunctionMayNotThrow = functionDecl(isNoThrow()); - Finder->addMatcher(cxxNewExpr(matchers::mayThrow(), + Finder->addMatcher(cxxNewExpr(mayThrow(), unless(hasAncestor(BadAllocCatchingTryBlock)), hasAncestor(FunctionMayNotThrow)) .bind("new-expr"), diff --git a/clang-tools-extra/clang-tidy/bugprone/VirtualNearMissCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/VirtualNearMissCheck.cpp index bd73a7cac378..bebdce525e28 100644 --- a/clang-tools-extra/clang-tidy/bugprone/VirtualNearMissCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/VirtualNearMissCheck.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "VirtualNearMissCheck.h" -#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/AST/CXXInheritance.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -18,6 +17,8 @@ using namespace clang::ast_matchers; namespace clang::tidy::bugprone { namespace { +AST_MATCHER(CXXMethodDecl, isStatic) { return Node.isStatic(); } + AST_MATCHER(CXXMethodDecl, isOverloadedOperator) { return Node.isOverloadedOperator(); } @@ -215,8 +216,8 @@ void VirtualNearMissCheck::registerMatchers(MatchFinder *Finder) { Finder->addMatcher( cxxMethodDecl( unless(anyOf(isOverride(), isImplicit(), cxxConstructorDecl(), - cxxDestructorDecl(), cxxConversionDecl(), - matchers::isStaticMethod(), isOverloadedOperator()))) + cxxDestructorDecl(), cxxConversionDecl(), isStatic(), + isOverloadedOperator()))) .bind("method"), this); } diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.cpp index d933c043da64..3923df312791 100644 --- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.cpp +++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.cpp @@ -20,9 +20,7 @@ using namespace clang::ast_matchers; namespace clang::tidy::cppcoreguidelines { -const clang::ast_matchers::internal::VariadicDynCastAllOfMatcher - VAArgExpr; +const internal::VariadicDynCastAllOfMatcher VAArgExpr; static constexpr StringRef AllowedVariadics[] = { // clang-format off diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp index a93b603099b8..aa70b3896f16 100644 --- a/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp +++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp @@ -18,7 +18,6 @@ using namespace clang::ast_matchers; namespace clang::tidy::cppcoreguidelines { -namespace { AST_MATCHER(CXXRecordDecl, hasPublicVirtualOrProtectedNonVirtualDestructor) { // We need to call Node.getDestructor() instead of matching a // CXXDestructorDecl. Otherwise, tests will fail for class templates, since @@ -34,7 +33,6 @@ AST_MATCHER(CXXRecordDecl, hasPublicVirtualOrProtectedNonVirtualDestructor) { !Destructor->isVirtual())); } -} // namespace void VirtualClassDestructorCheck::registerMatchers(MatchFinder *Finder) { ast_matchers::internal::Matcher InheritsVirtualMethod = hasAnyBase(hasType(cxxRecordDecl(has(cxxMethodDecl(isVirtual()))))); diff --git a/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp b/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp index c7590b515232..805dcaf3ce40 100644 --- a/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp +++ b/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp @@ -234,7 +234,7 @@ static bool isInInstantiation(const NodeType &Node, template static bool isInTemplate(const NodeType &Node, const MatchFinder::MatchResult &Result) { - ast_matchers::internal::Matcher IsInsideTemplate = + internal::Matcher IsInsideTemplate = hasAncestor(decl(anyOf(classTemplateDecl(), functionTemplateDecl()))); return !match(IsInsideTemplate, Node, *Result.Context).empty(); } diff --git a/clang-tools-extra/clang-tidy/llvmlibc/NamespaceConstants.h b/clang-tools-extra/clang-tidy/llvmlibc/NamespaceConstants.h index 77d4bd0a2541..7d4120085b86 100644 --- a/clang-tools-extra/clang-tidy/llvmlibc/NamespaceConstants.h +++ b/clang-tools-extra/clang-tidy/llvmlibc/NamespaceConstants.h @@ -6,9 +6,6 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_LLVMLIBC_NAMESPACE_CONSTANTS_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_LLVMLIBC_NAMESPACE_CONSTANTS_H - #include "llvm/ADT/StringRef.h" namespace clang::tidy::llvm_libc { @@ -17,5 +14,3 @@ const static llvm::StringRef RequiredNamespaceStart = "__llvm_libc"; const static llvm::StringRef RequiredNamespaceMacroName = "LIBC_NAMESPACE"; } // namespace clang::tidy::llvm_libc - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_LLVMLIBC_NAMESPACE_CONSTANTS_H diff --git a/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp b/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp index 517288930fa8..89790ea70cf2 100644 --- a/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp @@ -20,6 +20,14 @@ AST_MATCHER(clang::TypeLoc, hasValidBeginLoc) { return Node.getBeginLoc().isValid(); } +AST_MATCHER_P(clang::TypeLoc, hasType, + clang::ast_matchers::internal::Matcher, + InnerMatcher) { + const clang::Type *TypeNode = Node.getTypePtr(); + return TypeNode != nullptr && + InnerMatcher.matches(*TypeNode, Finder, Builder); +} + AST_MATCHER(clang::RecordDecl, isExternCContext) { return Node.isExternCContext(); } @@ -51,7 +59,7 @@ void AvoidCArraysCheck::registerMatchers(MatchFinder *Finder) { unless(parmVarDecl()))))); Finder->addMatcher( - typeLoc(hasValidBeginLoc(), loc(arrayType()), + typeLoc(hasValidBeginLoc(), hasType(arrayType()), unless(anyOf(hasParent(parmVarDecl(isArgvOfMain())), hasParent(varDecl(isExternC())), hasParent(fieldDecl( diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp index 9b05c3a7425c..3229e302eb43 100644 --- a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp @@ -94,7 +94,7 @@ static StatementMatcher incrementVarMatcher() { } static StatementMatcher -arrayConditionMatcher(ast_matchers::internal::Matcher LimitExpr) { +arrayConditionMatcher(internal::Matcher LimitExpr) { return binaryOperator( anyOf(allOf(hasOperatorName("<"), hasLHS(integerComparisonMatcher()), hasRHS(LimitExpr)), @@ -209,7 +209,7 @@ StatementMatcher makeIteratorLoopMatcher(bool IsReverse) { // This matcher tests that a declaration is a CXXRecordDecl that has an // overloaded operator*(). If the operator*() returns by value instead of by // reference then the return type is tagged with DerefByValueResultName. - ast_matchers::internal::Matcher TestDerefReturnsByValue = + internal::Matcher TestDerefReturnsByValue = hasType(hasUnqualifiedDesugaredType( recordType(hasDeclaration(cxxRecordDecl(hasMethod(cxxMethodDecl( hasOverloadedOperatorName("*"), diff --git a/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp index dc8a87b8ae0f..aec67808846b 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "UseAutoCheck.h" -#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/AST/TypeLoc.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -21,7 +20,6 @@ using namespace clang::ast_matchers; using namespace clang::ast_matchers::internal; namespace clang::tidy::modernize { - namespace { const char IteratorDeclStmtId[] = "iterator_decl"; @@ -157,6 +155,14 @@ Matcher hasStdContainerName() { return hasAnyName(ContainerNames); } +/// Matches declaration reference or member expressions with explicit template +/// arguments. +AST_POLYMORPHIC_MATCHER(hasExplicitTemplateArgs, + AST_POLYMORPHIC_SUPPORTED_TYPES(DeclRefExpr, + MemberExpr)) { + return Node.hasExplicitTemplateArgs(); +} + /// Returns a DeclarationMatcher that matches standard iterators nested /// inside records with a standard container name. DeclarationMatcher standardIterator() { @@ -232,9 +238,9 @@ StatementMatcher makeDeclWithTemplateCastMatcher() { auto ST = substTemplateTypeParmType(hasReplacementType(equalsBoundNode("arg"))); - auto ExplicitCall = anyOf( - has(memberExpr(matchers::hasExplicitTemplateArgs())), - has(ignoringImpCasts(declRefExpr(matchers::hasExplicitTemplateArgs())))); + auto ExplicitCall = + anyOf(has(memberExpr(hasExplicitTemplateArgs())), + has(ignoringImpCasts(declRefExpr(hasExplicitTemplateArgs())))); auto TemplateArg = hasTemplateArgument(0, refersToType(qualType().bind("arg"))); diff --git a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp index c477e566c787..430455a38f39 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp @@ -7,9 +7,7 @@ //===----------------------------------------------------------------------===// #include "UseEmplaceCheck.h" -#include "../utils/Matchers.h" #include "../utils/OptionsUtils.h" - using namespace clang::ast_matchers; namespace clang::tidy::modernize { @@ -80,6 +78,10 @@ AST_MATCHER(CXXMemberCallExpr, hasSameNumArgsAsDeclNumParams) { return Node.getNumArgs() == Node.getMethodDecl()->getNumParams(); } +AST_MATCHER(DeclRefExpr, hasExplicitTemplateArgs) { + return Node.hasExplicitTemplateArgs(); +} + // Helper Matcher which applies the given QualType Matcher either directly or by // resolving a pointer type to its pointee. Used to match v.push_back() as well // as p->push_back(). @@ -219,7 +221,7 @@ void UseEmplaceCheck::registerMatchers(MatchFinder *Finder) { auto MakeTuple = ignoringImplicit( callExpr(callee(expr(ignoringImplicit(declRefExpr( - unless(matchers::hasExplicitTemplateArgs()), + unless(hasExplicitTemplateArgs()), to(functionDecl(hasAnyName(TupleMakeFunctions)))))))) .bind("make")); diff --git a/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp index 406be8c66be7..5134eb51a032 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp @@ -48,8 +48,8 @@ static std::set getAllDirectBases(const CXXRecordDecl *Record) { /// Returns a matcher that matches member expressions where the base is /// the variable declared as \p Var and the accessed member is the one declared /// as \p Field. -ast_matchers::internal::Matcher accessToFieldInVar(const FieldDecl *Field, - const ValueDecl *Var) { +internal::Matcher accessToFieldInVar(const FieldDecl *Field, + const ValueDecl *Var) { return ignoringImpCasts( memberExpr(hasObjectExpression(declRefExpr(to(varDecl(equalsNode(Var))))), member(fieldDecl(equalsNode(Field))))); diff --git a/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp index 9fdd60ead040..9561cc71183d 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp @@ -16,7 +16,6 @@ using namespace clang::ast_matchers; namespace clang::tidy::modernize { namespace { - AST_MATCHER(FunctionDecl, hasAnyDefinition) { if (Node.hasBody() || Node.isPureVirtual() || Node.isDefaulted() || Node.isDeleted()) @@ -42,6 +41,9 @@ AST_MATCHER(CXXMethodDecl, isSpecialFunction) { } } // namespace +static const char SpecialFunction[] = "SpecialFunction"; +static const char DeletedNotPublic[] = "DeletedNotPublic"; + UseEqualsDeleteCheck::UseEqualsDeleteCheck(StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), @@ -61,17 +63,17 @@ void UseEqualsDeleteCheck::registerMatchers(MatchFinder *Finder) { // defined. unless(ofClass(hasMethod(cxxMethodDecl(unless(PrivateSpecialFn), unless(hasAnyDefinition())))))) - .bind("SpecialFunction"), + .bind(SpecialFunction), this); Finder->addMatcher( - cxxMethodDecl(isDeleted(), unless(isPublic())).bind("DeletedNotPublic"), + cxxMethodDecl(isDeleted(), unless(isPublic())).bind(DeletedNotPublic), this); } void UseEqualsDeleteCheck::check(const MatchFinder::MatchResult &Result) { if (const auto *Func = - Result.Nodes.getNodeAs("SpecialFunction")) { + Result.Nodes.getNodeAs(SpecialFunction)) { SourceLocation EndLoc = Lexer::getLocForEndOfToken( Func->getEndLoc(), 0, *Result.SourceManager, getLangOpts()); @@ -82,7 +84,7 @@ void UseEqualsDeleteCheck::check(const MatchFinder::MatchResult &Result) { "use '= delete' to prohibit calling of a special member function") << FixItHint::CreateInsertion(EndLoc, " = delete"); } else if (const auto *Func = - Result.Nodes.getNodeAs("DeletedNotPublic")) { + Result.Nodes.getNodeAs(DeletedNotPublic)) { // Ignore this warning in macros, since it's extremely noisy in code using // DISALLOW_COPY_AND_ASSIGN-style macros and there's no easy way to // automatically fix the warning when macros are in play. diff --git a/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp index 225a1e7bedea..2223a1999f73 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp @@ -62,7 +62,7 @@ void UseTransparentFunctorsCheck::registerMatchers(MatchFinder *Finder) { this); } -static const StringRef MessageDiag = "prefer transparent functors '%0<>'"; +static const StringRef Message = "prefer transparent functors '%0<>'"; template static T getInnerTypeLocAs(TypeLoc Loc) { T Result; @@ -79,7 +79,7 @@ void UseTransparentFunctorsCheck::check( Result.Nodes.getNodeAs("FunctorClass"); if (const auto *FuncInst = Result.Nodes.getNodeAs("FuncInst")) { - diag(FuncInst->getBeginLoc(), MessageDiag) << FuncClass->getName(); + diag(FuncInst->getBeginLoc(), Message) << FuncClass->getName(); return; } @@ -117,9 +117,9 @@ void UseTransparentFunctorsCheck::check( SourceLocation ReportLoc = FunctorLoc.getLocation(); if (ReportLoc.isInvalid()) return; - diag(ReportLoc, MessageDiag) - << FuncClass->getName() - << FixItHint::CreateRemoval(FunctorTypeLoc.getArgLoc(0).getSourceRange()); + diag(ReportLoc, Message) << FuncClass->getName() + << FixItHint::CreateRemoval( + FunctorTypeLoc.getArgLoc(0).getSourceRange()); } } // namespace clang::tidy::modernize diff --git a/clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp b/clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp index 103268caf32c..42f383edc67e 100644 --- a/clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp +++ b/clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "MissingHashCheck.h" -#include "ObjcMatcher.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -17,6 +16,12 @@ namespace clang::tidy::objc { namespace { +AST_MATCHER_P(ObjCImplementationDecl, hasInterface, + ast_matchers::internal::Matcher, Base) { + const ObjCInterfaceDecl *InterfaceDecl = Node.getClassInterface(); + return Base.matches(*InterfaceDecl, Finder, Builder); +} + AST_MATCHER_P(ObjCContainerDecl, hasInstanceMethod, ast_matchers::internal::Matcher, Base) { // Check each instance method against the provided matcher. diff --git a/clang-tools-extra/clang-tidy/objc/ObjcMatcher.h b/clang-tools-extra/clang-tidy/objc/ObjcMatcher.h deleted file mode 100644 index f272b5b0f8a6..000000000000 --- a/clang-tools-extra/clang-tidy/objc/ObjcMatcher.h +++ /dev/null @@ -1,40 +0,0 @@ -//===- AbseilMatcher.h - clang-tidy ---------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_OBJCMATCHER_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_OBJCMATCHER_H - -#include "clang/AST/ASTContext.h" -#include "clang/ASTMatchers/ASTMatchFinder.h" - -namespace clang::ast_matchers { - -/// Matches Objective-C implementations with interfaces that match -/// \c Base. -/// -/// Example matches implementation declarations for X. -/// (matcher = objcImplementationDecl(hasInterface(hasName("X")))) -/// \code -/// @interface X -/// @end -/// @implementation X -/// @end -/// @interface Y -// @end -/// @implementation Y -/// @end -/// \endcode -AST_MATCHER_P(ObjCImplementationDecl, hasInterface, - ast_matchers::internal::Matcher, Base) { - const ObjCInterfaceDecl *InterfaceDecl = Node.getClassInterface(); - return Base.matches(*InterfaceDecl, Finder, Builder); -} - -} // namespace clang::ast_matchers - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_OBJCMATCHER_H diff --git a/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.cpp b/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.cpp index 572a1463e5c0..951cbc52c9a9 100644 --- a/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.cpp +++ b/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "SuperSelfCheck.h" -#include "ObjcMatcher.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -33,6 +32,27 @@ AST_MATCHER(ObjCMethodDecl, isInitializer) { return Node.getMethodFamily() == OMF_init; } +/// Matches Objective-C implementations with interfaces that match +/// \c Base. +/// +/// Example matches implementation declarations for X. +/// (matcher = objcImplementationDecl(hasInterface(hasName("X")))) +/// \code +/// @interface X +/// @end +/// @implementation X +/// @end +/// @interface Y +// @end +/// @implementation Y +/// @end +/// \endcode +AST_MATCHER_P(ObjCImplementationDecl, hasInterface, + ast_matchers::internal::Matcher, Base) { + const ObjCInterfaceDecl *InterfaceDecl = Node.getClassInterface(); + return Base.matches(*InterfaceDecl, Finder, Builder); +} + /// Matches Objective-C message expressions where the receiver is the /// super instance. /// diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp index 8c38bad593a3..9beb185cba92 100644 --- a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp +++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp @@ -232,7 +232,7 @@ UnnecessaryCopyInitialization::UnnecessaryCopyInitialization( Options.get("ExcludedContainerTypes", ""))) {} void UnnecessaryCopyInitialization::registerMatchers(MatchFinder *Finder) { - auto LocalVarCopiedFrom = [this](const auto &CopyCtorArg) { + auto LocalVarCopiedFrom = [this](const internal::Matcher &CopyCtorArg) { return compoundStmt( forEachDescendant( declStmt( diff --git a/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp b/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp index 5b1bf0108cbb..1284df6bd99c 100644 --- a/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp +++ b/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "ConvertMemberFunctionsToStatic.h" -#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/RecursiveASTVisitor.h" @@ -19,12 +18,40 @@ using namespace clang::ast_matchers; namespace clang::tidy::readability { -namespace { +AST_MATCHER(CXXMethodDecl, isStatic) { return Node.isStatic(); } + +AST_MATCHER(CXXMethodDecl, hasTrivialBody) { return Node.hasTrivialBody(); } AST_MATCHER(CXXMethodDecl, isOverloadedOperator) { return Node.isOverloadedOperator(); } +AST_MATCHER(CXXRecordDecl, hasAnyDependentBases) { + return Node.hasAnyDependentBases(); +} + +AST_MATCHER(CXXMethodDecl, isTemplate) { + return Node.getTemplatedKind() != FunctionDecl::TK_NonTemplate; +} + +AST_MATCHER(CXXMethodDecl, isDependentContext) { + return Node.isDependentContext(); +} + +AST_MATCHER(CXXMethodDecl, isInsideMacroDefinition) { + const ASTContext &Ctxt = Finder->getASTContext(); + return clang::Lexer::makeFileCharRange( + clang::CharSourceRange::getCharRange( + Node.getTypeSourceInfo()->getTypeLoc().getSourceRange()), + Ctxt.getSourceManager(), Ctxt.getLangOpts()) + .isInvalid(); +} + +AST_MATCHER_P(CXXMethodDecl, hasCanonicalDecl, + ast_matchers::internal::Matcher, InnerMatcher) { + return InnerMatcher.matches(*Node.getCanonicalDecl(), Finder, Builder); +} + AST_MATCHER(CXXMethodDecl, usesThis) { class FindUsageOfThis : public RecursiveASTVisitor { public: @@ -47,27 +74,22 @@ AST_MATCHER(CXXMethodDecl, usesThis) { return UsageOfThis.Used; } -} // namespace - void ConvertMemberFunctionsToStatic::registerMatchers(MatchFinder *Finder) { Finder->addMatcher( cxxMethodDecl( isDefinition(), isUserProvided(), unless(anyOf( - isExpansionInSystemHeader(), isVirtual(), - matchers::isStaticMethod(), matchers::hasTrivialBody(), - isOverloadedOperator(), cxxConstructorDecl(), cxxDestructorDecl(), - cxxConversionDecl(), matchers::isTemplate(), - matchers::isDependentContext(), + isExpansionInSystemHeader(), isVirtual(), isStatic(), + hasTrivialBody(), isOverloadedOperator(), cxxConstructorDecl(), + cxxDestructorDecl(), cxxConversionDecl(), isTemplate(), + isDependentContext(), ofClass(anyOf( isLambda(), - matchers::hasAnyDependentBases()) // Method might become - // virtual depending on - // template base class. + hasAnyDependentBases()) // Method might become virtual + // depending on template base class. ), - matchers::isInsideMacroDefinition(), - matchers::hasCanonicalDecl(matchers::isInsideMacroDefinition()), - usesThis()))) + isInsideMacroDefinition(), + hasCanonicalDecl(isInsideMacroDefinition()), usesThis()))) .bind("x"), this); } diff --git a/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp b/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp index 261c02ea2d53..759cdd44fd65 100644 --- a/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp @@ -202,9 +202,9 @@ void CognitiveComplexity::account(SourceLocation Loc, unsigned short Nesting, Total += Increase; } -class CognitiveComplexityFunctionASTVisitor final - : public RecursiveASTVisitor { - using Base = RecursiveASTVisitor; +class FunctionASTVisitor final + : public RecursiveASTVisitor { + using Base = RecursiveASTVisitor; // If set to true, macros are ignored during analysis. const bool IgnoreMacros; @@ -219,7 +219,7 @@ class CognitiveComplexityFunctionASTVisitor final std::stack> BinaryOperatorsStack; public: - explicit CognitiveComplexityFunctionASTVisitor(const bool IgnoreMacros) + explicit FunctionASTVisitor(const bool IgnoreMacros) : IgnoreMacros(IgnoreMacros) {} bool traverseStmtWithIncreasedNestingLevel(Stmt *Node) { @@ -453,13 +453,12 @@ public: // The parameter MainAnalyzedFunction is needed to differentiate between the // cases where TraverseDecl() is the entry point from // FunctionCognitiveComplexityCheck::check() and the cases where it was called - // from the CognitiveComplexityFunctionASTVisitor itself. Explanation: if we - // get a function definition (e.g. constructor, destructor, method), the - // Cognitive Complexity specification states that the Nesting level shall be - // increased. But if this function is the entry point, then the Nesting level - // should not be increased. Thus that parameter is there and is used to - // fall-through directly to traversing if this is the main function that is - // being analyzed. + // from the FunctionASTVisitor itself. Explanation: if we get a function + // definition (e.g. constructor, destructor, method), the Cognitive Complexity + // specification states that the Nesting level shall be increased. But if this + // function is the entry point, then the Nesting level should not be + // increased. Thus that parameter is there and is used to fall-through + // directly to traversing if this is the main function that is being analyzed. bool TraverseDecl(Decl *Node, bool MainAnalyzedFunction = false) { if (!Node || MainAnalyzedFunction) return Base::TraverseDecl(Node); @@ -516,7 +515,7 @@ void FunctionCognitiveComplexityCheck::registerMatchers(MatchFinder *Finder) { void FunctionCognitiveComplexityCheck::check( const MatchFinder::MatchResult &Result) { - CognitiveComplexityFunctionASTVisitor Visitor(IgnoreMacros); + FunctionASTVisitor Visitor(IgnoreMacros); SourceLocation Loc; const auto *TheDecl = Result.Nodes.getNodeAs("func"); diff --git a/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp b/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp index c828ef3bc78b..d42fcba70e81 100644 --- a/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "MakeMemberFunctionConstCheck.h" -#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/AST/ParentMapContext.h" #include "clang/AST/RecursiveASTVisitor.h" @@ -18,6 +17,36 @@ using namespace clang::ast_matchers; namespace clang::tidy::readability { +AST_MATCHER(CXXMethodDecl, isStatic) { return Node.isStatic(); } + +AST_MATCHER(CXXMethodDecl, hasTrivialBody) { return Node.hasTrivialBody(); } + +AST_MATCHER(CXXRecordDecl, hasAnyDependentBases) { + return Node.hasAnyDependentBases(); +} + +AST_MATCHER(CXXMethodDecl, isTemplate) { + return Node.getTemplatedKind() != FunctionDecl::TK_NonTemplate; +} + +AST_MATCHER(CXXMethodDecl, isDependentContext) { + return Node.isDependentContext(); +} + +AST_MATCHER(CXXMethodDecl, isInsideMacroDefinition) { + const ASTContext &Ctxt = Finder->getASTContext(); + return clang::Lexer::makeFileCharRange( + clang::CharSourceRange::getCharRange( + Node.getTypeSourceInfo()->getTypeLoc().getSourceRange()), + Ctxt.getSourceManager(), Ctxt.getLangOpts()) + .isInvalid(); +} + +AST_MATCHER_P(CXXMethodDecl, hasCanonicalDecl, + ast_matchers::internal::Matcher, InnerMatcher) { + return InnerMatcher.matches(*Node.getCanonicalDecl(), Finder, Builder); +} + enum UsageKind { Unused, Const, NonConst }; class FindUsageOfThis : public RecursiveASTVisitor { @@ -176,7 +205,6 @@ public: } }; -namespace { AST_MATCHER(CXXMethodDecl, usesThisAsConst) { FindUsageOfThis UsageOfThis(Finder->getASTContext()); @@ -186,8 +214,6 @@ AST_MATCHER(CXXMethodDecl, usesThisAsConst) { return UsageOfThis.Usage == Const; } -} // namespace - void MakeMemberFunctionConstCheck::registerMatchers(MatchFinder *Finder) { Finder->addMatcher( traverse( @@ -196,18 +222,15 @@ void MakeMemberFunctionConstCheck::registerMatchers(MatchFinder *Finder) { isDefinition(), isUserProvided(), unless(anyOf( isExpansionInSystemHeader(), isVirtual(), isConst(), - matchers::isStaticMethod(), matchers::hasTrivialBody(), - cxxConstructorDecl(), cxxDestructorDecl(), - matchers::isTemplate(), matchers::isDependentContext(), - ofClass(anyOf( - isLambda(), - matchers::hasAnyDependentBases()) // Method might become + isStatic(), hasTrivialBody(), cxxConstructorDecl(), + cxxDestructorDecl(), isTemplate(), isDependentContext(), + ofClass(anyOf(isLambda(), + hasAnyDependentBases()) // Method might become // virtual depending on // template base class. ), - matchers::isInsideMacroDefinition(), - matchers::hasCanonicalDecl( - matchers::isInsideMacroDefinition()))), + isInsideMacroDefinition(), + hasCanonicalDecl(isInsideMacroDefinition()))), usesThisAsConst()) .bind("x")), this); diff --git a/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp index bfd4dca11e94..7850a6f29995 100644 --- a/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp @@ -15,13 +15,10 @@ using namespace clang::ast_matchers; namespace clang::tidy::readability { -namespace { AST_MATCHER(FunctionDecl, doesDeclarationForceExternallyVisibleDefinition) { return Node.doesDeclarationForceExternallyVisibleDefinition(); } -} // namespace - RedundantDeclarationCheck::RedundantDeclarationCheck(StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), diff --git a/clang-tools-extra/clang-tidy/readability/RedundantSmartptrGetCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantSmartptrGetCheck.cpp index 86d3c7e860c9..8837ac16e882 100644 --- a/clang-tools-extra/clang-tidy/readability/RedundantSmartptrGetCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/RedundantSmartptrGetCheck.cpp @@ -15,8 +15,7 @@ using namespace clang::ast_matchers; namespace clang::tidy::readability { namespace { -clang::ast_matchers::internal::Matcher -callToGet(const clang::ast_matchers::internal::Matcher &OnClass) { +internal::Matcher callToGet(const internal::Matcher &OnClass) { return expr( anyOf(cxxMemberCallExpr( on(expr(anyOf(hasType(OnClass), @@ -44,7 +43,7 @@ callToGet(const clang::ast_matchers::internal::Matcher &OnClass) { .bind("redundant_get"); } -clang::ast_matchers::internal::Matcher knownSmartptr() { +internal::Matcher knownSmartptr() { return recordDecl(hasAnyName("::std::unique_ptr", "::std::shared_ptr")); } diff --git a/clang-tools-extra/clang-tidy/readability/ReferenceToConstructedTemporaryCheck.cpp b/clang-tools-extra/clang-tidy/readability/ReferenceToConstructedTemporaryCheck.cpp index 9abe0f0e155a..587ae8ea3058 100644 --- a/clang-tools-extra/clang-tidy/readability/ReferenceToConstructedTemporaryCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/ReferenceToConstructedTemporaryCheck.cpp @@ -20,8 +20,7 @@ namespace { // Predicate structure to check if lifetime of temporary is not extended by // ValueDecl pointed out by ID struct NotExtendedByDeclBoundToPredicate { - bool - operator()(const clang::ast_matchers::internal::BoundNodesMap &Nodes) const { + bool operator()(const internal::BoundNodesMap &Nodes) const { const auto *Other = Nodes.getNodeAs(ID); if (!Other) return true; diff --git a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp index 77b8f5e5ac21..65356cc3929c 100644 --- a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "StaticAccessedThroughInstanceCheck.h" -#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" #include "llvm/ADT/StringRef.h" @@ -16,6 +15,10 @@ using namespace clang::ast_matchers; namespace clang::tidy::readability { +namespace { +AST_MATCHER(CXXMethodDecl, isStatic) { return Node.isStatic(); } +} // namespace + static unsigned getNameSpecifierNestingLevel(const QualType &QType) { if (const auto *ElType = QType->getAs()) { if (const NestedNameSpecifier *NestedSpecifiers = ElType->getQualifier()) { @@ -39,7 +42,7 @@ void StaticAccessedThroughInstanceCheck::storeOptions( void StaticAccessedThroughInstanceCheck::registerMatchers(MatchFinder *Finder) { Finder->addMatcher( - memberExpr(hasDeclaration(anyOf(cxxMethodDecl(matchers::isStaticMethod()), + memberExpr(hasDeclaration(anyOf(cxxMethodDecl(isStatic()), varDecl(hasStaticStorageDuration()), enumConstantDecl()))) .bind("memberExpression"), diff --git a/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp b/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp index 8ee148a5bb39..df2b0bef576c 100644 --- a/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp +++ b/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp @@ -285,11 +285,4 @@ SourceLocation getLocationForNoexceptSpecifier(const FunctionDecl *FuncDecl, return {}; } -bool insideMacroDefinition(SourceRange Range, const SourceManager &SM, - const LangOptions &LangOpts) { - return clang::Lexer::makeFileCharRange( - clang::CharSourceRange::getCharRange(Range), SM, LangOpts) - .isInvalid(); -} - } // namespace clang::tidy::utils::lexer diff --git a/clang-tools-extra/clang-tidy/utils/LexerUtils.h b/clang-tools-extra/clang-tidy/utils/LexerUtils.h index a697e7ee8adf..ea9bd512b68b 100644 --- a/clang-tools-extra/clang-tidy/utils/LexerUtils.h +++ b/clang-tools-extra/clang-tidy/utils/LexerUtils.h @@ -125,11 +125,6 @@ SourceLocation getUnifiedEndLoc(const Stmt &S, const SourceManager &SM, SourceLocation getLocationForNoexceptSpecifier(const FunctionDecl *FuncDecl, const SourceManager &SM); -// Returns true if a part of the range resides inside a macro expansion or the -// range does not reside on the same FileID. -bool insideMacroDefinition(SourceRange Range, const SourceManager &SM, - const LangOptions &LangOpts); - } // namespace tidy::utils::lexer } // namespace clang diff --git a/clang-tools-extra/clang-tidy/utils/Matchers.h b/clang-tools-extra/clang-tidy/utils/Matchers.h index dc47bb515e0e..045e3ffbb6a8 100644 --- a/clang-tools-extra/clang-tidy/utils/Matchers.h +++ b/clang-tools-extra/clang-tidy/utils/Matchers.h @@ -9,7 +9,6 @@ #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_MATCHERS_H #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_MATCHERS_H -#include "LexerUtils.h" #include "TypeTraits.h" #include "clang/AST/ExprConcepts.h" #include "clang/ASTMatchers/ASTMatchers.h" @@ -23,10 +22,6 @@ AST_MATCHER(BinaryOperator, isRelationalOperator) { AST_MATCHER(BinaryOperator, isEqualityOperator) { return Node.isEqualityOp(); } -AST_MATCHER_P(IntegerLiteral, isBiggerThan, unsigned, N) { - return Node.getValue().getZExtValue() > N; -} - AST_MATCHER(QualType, isExpensiveToCopy) { std::optional IsExpensive = utils::type_traits::isExpensiveToCopy(Node, Finder->getASTContext()); @@ -42,67 +37,6 @@ AST_MATCHER(QualType, isTriviallyDestructible) { return utils::type_traits::isTriviallyDestructible(Node); } -AST_MATCHER(CXXRecordDecl, hasAnyDependentBases) { - return Node.hasAnyDependentBases(); -} - -AST_MATCHER(CXXMethodDecl, isStaticMethod) { return Node.isStatic(); } - -AST_MATCHER(CXXMethodDecl, hasTrivialBody) { return Node.hasTrivialBody(); } - -AST_MATCHER(CXXMethodDecl, isTemplate) { - return Node.getTemplatedKind() != FunctionDecl::TK_NonTemplate; -} - -AST_MATCHER(CXXMethodDecl, isDependentContext) { - return Node.isDependentContext(); -} - -AST_MATCHER(CXXMethodDecl, isInsideMacroDefinition) { - const ASTContext &Ctxt = Finder->getASTContext(); - return utils::lexer::insideMacroDefinition( - Node.getTypeSourceInfo()->getTypeLoc().getSourceRange(), - Ctxt.getSourceManager(), Ctxt.getLangOpts()); -} - -AST_MATCHER_P(CXXMethodDecl, hasCanonicalDecl, - ast_matchers::internal::Matcher, InnerMatcher) { - return InnerMatcher.matches(*Node.getCanonicalDecl(), Finder, Builder); -} - -AST_MATCHER(CXXNewExpr, mayThrow) { - FunctionDecl *OperatorNew = Node.getOperatorNew(); - if (!OperatorNew) - return false; - if (auto *FuncType = OperatorNew->getType()->getAs()) - return !FuncType->isNothrow(); - return false; -} - -AST_MATCHER_P(CXXTryStmt, hasHandlerFor, - ast_matchers::internal::Matcher, InnerMatcher) { - for (unsigned NH = Node.getNumHandlers(), I = 0; I < NH; ++I) { - const CXXCatchStmt *CatchS = Node.getHandler(I); - // Check for generic catch handler (match anything). - if (CatchS->getCaughtType().isNull()) - return true; - ast_matchers::internal::BoundNodesTreeBuilder Result(*Builder); - if (InnerMatcher.matches(CatchS->getCaughtType(), Finder, &Result)) { - *Builder = std::move(Result); - return true; - } - } - return false; -} - -// Matches declaration reference or member expressions with explicit template -// arguments. -AST_POLYMORPHIC_MATCHER(hasExplicitTemplateArgs, - AST_POLYMORPHIC_SUPPORTED_TYPES(DeclRefExpr, - MemberExpr)) { - return Node.hasExplicitTemplateArgs(); -} - // Returns QualType matcher for references to const. AST_MATCHER_FUNCTION(ast_matchers::TypeMatcher, isReferenceToConst) { using namespace ast_matchers; diff --git a/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp b/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp index 214f0fd43a62..4703ce126981 100644 --- a/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp +++ b/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp @@ -13,23 +13,18 @@ #include "clang/ASTMatchers/ASTMatchers.h" #include "clang/Lex/Lexer.h" #include - namespace clang::tidy::utils { using namespace ast_matchers; -namespace { +NamespaceAliaser::NamespaceAliaser(const SourceManager &SourceMgr) + : SourceMgr(SourceMgr) {} AST_MATCHER_P(NamespaceAliasDecl, hasTargetNamespace, ast_matchers::internal::Matcher, innerMatcher) { return innerMatcher.matches(*Node.getNamespace(), Finder, Builder); } -} // namespace - -NamespaceAliaser::NamespaceAliaser(const SourceManager &SourceMgr) - : SourceMgr(SourceMgr) {} - std::optional NamespaceAliaser::createAlias(ASTContext &Context, const Stmt &Statement, StringRef Namespace, -- GitLab From 02078199d9839a8e0232a543b7bcd74867ca7d8e Mon Sep 17 00:00:00 2001 From: Jacek Caban Date: Sun, 31 Mar 2024 17:39:58 +0200 Subject: [PATCH 0063/1690] [IR][Object][NFC] Move ARM64EC name mangling helpers to Mangler.h. (#87191) Move the implementation to llvm:Core to avoids including `COFF.h` in `Mangler.cpp`. --- llvm/include/llvm/IR/Mangler.h | 4 ++ llvm/include/llvm/Object/COFF.h | 41 ------------------ llvm/include/llvm/Object/COFFImportFile.h | 1 + llvm/lib/IR/Mangler.cpp | 42 ++++++++++++++++++- .../AArch64/AArch64Arm64ECCallLowering.cpp | 2 +- 5 files changed, 46 insertions(+), 44 deletions(-) diff --git a/llvm/include/llvm/IR/Mangler.h b/llvm/include/llvm/IR/Mangler.h index 747a4085235c..f28ffc961b6d 100644 --- a/llvm/include/llvm/IR/Mangler.h +++ b/llvm/include/llvm/IR/Mangler.h @@ -14,6 +14,7 @@ #define LLVM_IR_MANGLER_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringRef.h" namespace llvm { @@ -52,6 +53,9 @@ void emitLinkerFlagsForGlobalCOFF(raw_ostream &OS, const GlobalValue *GV, void emitLinkerFlagsForUsedCOFF(raw_ostream &OS, const GlobalValue *GV, const Triple &T, Mangler &M); +std::optional getArm64ECMangledFunctionName(StringRef Name); +std::optional getArm64ECDemangledFunctionName(StringRef Name); + } // End llvm namespace #endif diff --git a/llvm/include/llvm/Object/COFF.h b/llvm/include/llvm/Object/COFF.h index 2a5c3d8913b1..a548b2c15c5f 100644 --- a/llvm/include/llvm/Object/COFF.h +++ b/llvm/include/llvm/Object/COFF.h @@ -1362,47 +1362,6 @@ public: SectionStrippedError() { setErrorCode(object_error::section_stripped); } }; -inline std::optional -getArm64ECMangledFunctionName(StringRef Name) { - bool IsCppFn = Name[0] == '?'; - if (IsCppFn && Name.find("$$h") != std::string::npos) - return std::nullopt; - if (!IsCppFn && Name[0] == '#') - return std::nullopt; - - StringRef Prefix = "$$h"; - size_t InsertIdx = 0; - if (IsCppFn) { - InsertIdx = Name.find("@@"); - size_t ThreeAtSignsIdx = Name.find("@@@"); - if (InsertIdx != std::string::npos && InsertIdx != ThreeAtSignsIdx) { - InsertIdx += 2; - } else { - InsertIdx = Name.find("@"); - if (InsertIdx != std::string::npos) - InsertIdx++; - } - } else { - Prefix = "#"; - } - - return std::optional( - (Name.substr(0, InsertIdx) + Prefix + Name.substr(InsertIdx)).str()); -} - -inline std::optional -getArm64ECDemangledFunctionName(StringRef Name) { - if (Name[0] == '#') - return std::string(Name.substr(1)); - if (Name[0] != '?') - return std::nullopt; - - std::pair Pair = Name.split("$$h"); - if (Pair.second.empty()) - return std::nullopt; - return (Pair.first + Pair.second).str(); -} - } // end namespace object } // end namespace llvm diff --git a/llvm/include/llvm/Object/COFFImportFile.h b/llvm/include/llvm/Object/COFFImportFile.h index 7268faa87eb7..035bc17126da 100644 --- a/llvm/include/llvm/Object/COFFImportFile.h +++ b/llvm/include/llvm/Object/COFFImportFile.h @@ -17,6 +17,7 @@ #define LLVM_OBJECT_COFFIMPORTFILE_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/IR/Mangler.h" #include "llvm/Object/COFF.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Object/SymbolicFile.h" diff --git a/llvm/lib/IR/Mangler.cpp b/llvm/lib/IR/Mangler.cpp index c4e8f005d6c8..72e2bc1f24ac 100644 --- a/llvm/lib/IR/Mangler.cpp +++ b/llvm/lib/IR/Mangler.cpp @@ -18,7 +18,6 @@ #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/Module.h" -#include "llvm/Object/COFF.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TargetParser/Triple.h" @@ -242,7 +241,7 @@ void llvm::emitLinkerFlagsForGlobalCOFF(raw_ostream &OS, const GlobalValue *GV, // typically functions correctly; the linker can resolve the export // with the demangled alias. if (std::optional demangledName = - object::getArm64ECDemangledFunctionName(GV->getName())) + getArm64ECDemangledFunctionName(GV->getName())) OS << ",EXPORTAS," << *demangledName; } if (NeedQuotes) @@ -291,3 +290,42 @@ void llvm::emitLinkerFlagsForUsedCOFF(raw_ostream &OS, const GlobalValue *GV, OS << "\""; } +std::optional llvm::getArm64ECMangledFunctionName(StringRef Name) { + bool IsCppFn = Name[0] == '?'; + if (IsCppFn && Name.find("$$h") != std::string::npos) + return std::nullopt; + if (!IsCppFn && Name[0] == '#') + return std::nullopt; + + StringRef Prefix = "$$h"; + size_t InsertIdx = 0; + if (IsCppFn) { + InsertIdx = Name.find("@@"); + size_t ThreeAtSignsIdx = Name.find("@@@"); + if (InsertIdx != std::string::npos && InsertIdx != ThreeAtSignsIdx) { + InsertIdx += 2; + } else { + InsertIdx = Name.find("@"); + if (InsertIdx != std::string::npos) + InsertIdx++; + } + } else { + Prefix = "#"; + } + + return std::optional( + (Name.substr(0, InsertIdx) + Prefix + Name.substr(InsertIdx)).str()); +} + +std::optional +llvm::getArm64ECDemangledFunctionName(StringRef Name) { + if (Name[0] == '#') + return std::optional(Name.substr(1)); + if (Name[0] != '?') + return std::nullopt; + + std::pair Pair = Name.split("$$h"); + if (Pair.second.empty()) + return std::nullopt; + return std::optional((Pair.first + Pair.second).str()); +} diff --git a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp index 9b5cbe3de137..3bf6283b79e9 100644 --- a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp @@ -23,6 +23,7 @@ #include "llvm/IR/CallingConv.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" +#include "llvm/IR/Mangler.h" #include "llvm/InitializePasses.h" #include "llvm/Object/COFF.h" #include "llvm/Pass.h" @@ -31,7 +32,6 @@ using namespace llvm; using namespace llvm::COFF; -using namespace llvm::object; using OperandBundleDef = OperandBundleDefT; -- GitLab From 45f5fa2925142edf9a3d1648fac6c1f80b360175 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Sat, 30 Mar 2024 21:39:40 -0500 Subject: [PATCH 0064/1690] [IR] Add commutable matcher for `add nuw`; NFC Closes #87179 --- llvm/include/llvm/IR/PatternMatch.h | 16 ++++++++++++++-- llvm/unittests/IR/PatternMatch.cpp | 10 ++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 46372c78263a..92cb79d54afc 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -1185,7 +1185,7 @@ inline BinaryOp_match m_AShr(const LHS &L, } template + unsigned WrapFlags = 0, bool Commutable = false> struct OverflowingBinaryOp_match { LHS_t L; RHS_t R; @@ -1203,7 +1203,9 @@ struct OverflowingBinaryOp_match { if ((WrapFlags & OverflowingBinaryOperator::NoSignedWrap) && !Op->hasNoSignedWrap()) return false; - return L.match(Op->getOperand(0)) && R.match(Op->getOperand(1)); + return (L.match(Op->getOperand(0)) && R.match(Op->getOperand(1))) || + (Commutable && L.match(Op->getOperand(1)) && + R.match(Op->getOperand(0))); } return false; } @@ -1250,6 +1252,16 @@ m_NUWAdd(const LHS &L, const RHS &R) { OverflowingBinaryOperator::NoUnsignedWrap>( L, R); } + +template +inline OverflowingBinaryOp_match< + LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap, true> +m_c_NUWAdd(const LHS &L, const RHS &R) { + return OverflowingBinaryOp_match(L, R); +} + template inline OverflowingBinaryOp_match diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp index a0b873de2d58..4d0c2e4220fe 100644 --- a/llvm/unittests/IR/PatternMatch.cpp +++ b/llvm/unittests/IR/PatternMatch.cpp @@ -948,6 +948,16 @@ TEST_F(PatternMatchTest, OverflowingBinOps) { EXPECT_EQ(L, MatchL); EXPECT_EQ(R, MatchR); MatchL = MatchR = nullptr; + + EXPECT_TRUE( + m_c_NUWAdd(m_Specific(L), m_Specific(R)).match(IRB.CreateNUWAdd(L, R))); + EXPECT_TRUE( + m_c_NUWAdd(m_Specific(R), m_Specific(L)).match(IRB.CreateNUWAdd(L, R))); + EXPECT_FALSE( + m_c_NUWAdd(m_Specific(R), m_ZeroInt()).match(IRB.CreateNUWAdd(L, R))); + EXPECT_FALSE( + m_NUWAdd(m_Specific(R), m_Specific(L)).match(IRB.CreateNUWAdd(L, R))); + EXPECT_TRUE( m_NUWSub(m_Value(MatchL), m_Value(MatchR)).match(IRB.CreateNUWSub(L, R))); EXPECT_EQ(L, MatchL); -- GitLab From 154cea46732f4014bb409f1bcac9b39ac56df193 Mon Sep 17 00:00:00 2001 From: Pablo Busse Date: Sun, 31 Mar 2024 09:17:32 -0700 Subject: [PATCH 0065/1690] [lldb] Fix type lookup in DWARF .o files via debug map (#87177) An inverted condition causes `SymbolFileDWARFDebugMap::FindTypes` to bail out after inspecting the first .o file in each module. The same kind of bug is found in `SymbolFileDWARFDebugMap::ParseDeclsForContext`. Correct both early exit conditions and add a regression test for lookup of up a type defined in a secondary compilation unit. Fixes #87176 --- .../DWARF/SymbolFileDWARFDebugMap.cpp | 4 +-- .../functionalities/type_find_first/Makefile | 2 +- .../type_find_first/TestFindFirstType.py | 27 ++++++++++--------- .../functionalities/type_find_first/main.cpp | 5 ++++ .../functionalities/type_find_first/other.cpp | 4 +++ 5 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 lldb/test/API/functionalities/type_find_first/other.cpp diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp index 6dd3eb3677b7..4bc2cfd60688 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp @@ -1233,7 +1233,7 @@ void SymbolFileDWARFDebugMap::FindTypes(const TypeQuery &query, std::lock_guard guard(GetModuleMutex()); ForEachSymbolFile([&](SymbolFileDWARF *oso_dwarf) -> bool { oso_dwarf->FindTypes(query, results); - return !results.Done(query); // Keep iterating if we aren't done. + return results.Done(query); // Keep iterating if we aren't done. }); } @@ -1391,7 +1391,7 @@ void SymbolFileDWARFDebugMap::ParseDeclsForContext( lldb_private::CompilerDeclContext decl_ctx) { ForEachSymbolFile([&](SymbolFileDWARF *oso_dwarf) -> bool { oso_dwarf->ParseDeclsForContext(decl_ctx); - return true; // Keep iterating + return false; // Keep iterating }); } diff --git a/lldb/test/API/functionalities/type_find_first/Makefile b/lldb/test/API/functionalities/type_find_first/Makefile index 3d0b98f13f3d..e027553c7a6b 100644 --- a/lldb/test/API/functionalities/type_find_first/Makefile +++ b/lldb/test/API/functionalities/type_find_first/Makefile @@ -1,2 +1,2 @@ -CXX_SOURCES := main.cpp +CXX_SOURCES := main.cpp other.cpp include Makefile.rules diff --git a/lldb/test/API/functionalities/type_find_first/TestFindFirstType.py b/lldb/test/API/functionalities/type_find_first/TestFindFirstType.py index 6347a35e72ea..b1c5659a324a 100644 --- a/lldb/test/API/functionalities/type_find_first/TestFindFirstType.py +++ b/lldb/test/API/functionalities/type_find_first/TestFindFirstType.py @@ -8,8 +8,6 @@ from lldbsuite.test import lldbutil class TypeFindFirstTestCase(TestBase): - NO_DEBUG_INFO_TESTCASE = True - def test_find_first_type(self): """ Test SBTarget::FindFirstType() and SBModule::FindFirstType() APIs. @@ -19,19 +17,22 @@ class TypeFindFirstTestCase(TestBase): basename, FindFirstType() could end up failing depending on which type was found first in the debug info indexes. This test will ensure this doesn't regress in the future. + + The test also looks for a type defined in a different compilation unit + to verify that SymbolFileDWARFDebugMap searches each symbol file in a + module. """ self.build() target = self.createTestTarget() - # Test the SBTarget APIs for FindFirstType - integer_type = target.FindFirstType("Integer::Point") - self.assertTrue(integer_type.IsValid()) - float_type = target.FindFirstType("Float::Point") - self.assertTrue(float_type.IsValid()) - - # Test the SBModule APIs for FindFirstType exe_module = target.GetModuleAtIndex(0) self.assertTrue(exe_module.IsValid()) - integer_type = exe_module.FindFirstType("Integer::Point") - self.assertTrue(integer_type.IsValid()) - float_type = exe_module.FindFirstType("Float::Point") - self.assertTrue(float_type.IsValid()) + # Test the SBTarget and SBModule APIs for FindFirstType + for api in [target, exe_module]: + integer_type = api.FindFirstType("Integer::Point") + self.assertTrue(integer_type.IsValid()) + float_type = api.FindFirstType("Float::Point") + self.assertTrue(float_type.IsValid()) + external_type = api.FindFirstType("OtherCompilationUnit::Type") + self.assertTrue(external_type.IsValid()) + nonexistent_type = api.FindFirstType("NonexistentType") + self.assertFalse(nonexistent_type.IsValid()) diff --git a/lldb/test/API/functionalities/type_find_first/main.cpp b/lldb/test/API/functionalities/type_find_first/main.cpp index f4e467286004..bbb060872a1e 100644 --- a/lldb/test/API/functionalities/type_find_first/main.cpp +++ b/lldb/test/API/functionalities/type_find_first/main.cpp @@ -10,8 +10,13 @@ struct Point { }; } // namespace Float +namespace OtherCompilationUnit { +void Function(); +} // namespace OtherCompilationUnit + int main(int argc, char const *argv[]) { Integer::Point ip = {2, 3}; Float::Point fp = {2.0, 3.0}; + OtherCompilationUnit::Function(); return 0; } diff --git a/lldb/test/API/functionalities/type_find_first/other.cpp b/lldb/test/API/functionalities/type_find_first/other.cpp new file mode 100644 index 000000000000..b91edcd8e1d7 --- /dev/null +++ b/lldb/test/API/functionalities/type_find_first/other.cpp @@ -0,0 +1,4 @@ +namespace OtherCompilationUnit { +struct Type {}; +void Function() { Type typeIsActuallyUsed; } +} // namespace OtherCompilationUnit \ No newline at end of file -- GitLab From 8d8fff09d7e1267e75c2b9a1986c9d64533c177e Mon Sep 17 00:00:00 2001 From: Jacek Caban Date: Sun, 31 Mar 2024 18:30:04 +0200 Subject: [PATCH 0066/1690] [Object][COFF][NFC] Don't use inline function for COFFImportFile::printSymbolName. (#87195) Fixes BUILD_SHARED_LIBS builds after #87191 made helpers non-inline. --- llvm/include/llvm/Object/COFFImportFile.h | 21 +-------------------- llvm/lib/Object/COFFImportFile.cpp | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/llvm/include/llvm/Object/COFFImportFile.h b/llvm/include/llvm/Object/COFFImportFile.h index 035bc17126da..2c06f529ecdf 100644 --- a/llvm/include/llvm/Object/COFFImportFile.h +++ b/llvm/include/llvm/Object/COFFImportFile.h @@ -45,26 +45,7 @@ public: void moveSymbolNext(DataRefImpl &Symb) const override { ++Symb.p; } - Error printSymbolName(raw_ostream &OS, DataRefImpl Symb) const override { - switch (Symb.p) { - case ImpSymbol: - OS << "__imp_"; - break; - case ECAuxSymbol: - OS << "__imp_aux_"; - break; - } - const char *Name = Data.getBufferStart() + sizeof(coff_import_header); - if (Symb.p != ECThunkSymbol && COFF::isArm64EC(getMachine())) { - if (std::optional DemangledName = - getArm64ECDemangledFunctionName(Name)) { - OS << StringRef(*DemangledName); - return Error::success(); - } - } - OS << StringRef(Name); - return Error::success(); - } + Error printSymbolName(raw_ostream &OS, DataRefImpl Symb) const override; Expected getSymbolFlags(DataRefImpl Symb) const override { return SymbolRef::SF_Global; diff --git a/llvm/lib/Object/COFFImportFile.cpp b/llvm/lib/Object/COFFImportFile.cpp index 477c5bf98249..48c3ea0ed8f4 100644 --- a/llvm/lib/Object/COFFImportFile.cpp +++ b/llvm/lib/Object/COFFImportFile.cpp @@ -84,6 +84,27 @@ StringRef COFFImportFile::getExportName() const { return name; } +Error COFFImportFile::printSymbolName(raw_ostream &OS, DataRefImpl Symb) const { + switch (Symb.p) { + case ImpSymbol: + OS << "__imp_"; + break; + case ECAuxSymbol: + OS << "__imp_aux_"; + break; + } + const char *Name = Data.getBufferStart() + sizeof(coff_import_header); + if (Symb.p != ECThunkSymbol && COFF::isArm64EC(getMachine())) { + if (std::optional DemangledName = + getArm64ECDemangledFunctionName(Name)) { + OS << StringRef(*DemangledName); + return Error::success(); + } + } + OS << StringRef(Name); + return Error::success(); +} + static uint16_t getImgRelRelocation(MachineTypes Machine) { switch (Machine) { default: -- GitLab From 0234d90d818204f3a575de744a8df8448a7adeca Mon Sep 17 00:00:00 2001 From: Austin Kerbow Date: Sun, 31 Mar 2024 10:46:05 -0700 Subject: [PATCH 0067/1690] [AMDGPU] Extend MFMA padding option to gfx90a+ (#86768) It was shown experimentally that this may have some benefit on newer HW. --- .../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 3 + .../AMDGPU/neighboring-mfma-padding.mir | 504 ++++++++++++++++++ 2 files changed, 507 insertions(+) diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp index e515b729e7d7..75766b11ca82 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -2357,6 +2357,9 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { break; } + // Pad neighboring MFMA with noops for better inter-wave performance. + WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI)); + return WaitStatesNeeded; } diff --git a/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir b/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir index 3de258bb52a5..bf2cf6aeb990 100644 --- a/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir +++ b/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir @@ -5,6 +5,14 @@ # RUN: llc -mtriple=amdgcn -mcpu=gfx908 -amdgpu-mfma-padding-ratio=75 -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=gfx908-PAD75 %s # RUN: llc -mtriple=amdgcn -mcpu=gfx908 -amdgpu-mfma-padding-ratio=100 -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=gfx908-PAD100 %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=gfx90a-DEFAULT %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-padding-ratio=50 -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=gfx90a-PAD50 %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-padding-ratio=100 -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=gfx90a-PAD100 %s + +# RUN: llc -mtriple=amdgcn -mcpu=gfx940 -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=gfx940-DEFAULT %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx940 -amdgpu-mfma-padding-ratio=50 -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=gfx940-PAD50 %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx940 -amdgpu-mfma-padding-ratio=100 -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=gfx940-PAD100 %s + --- name: mfma_padding_2_pass body: | @@ -31,6 +39,35 @@ body: | ; gfx908-PAD100: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec ; gfx908-PAD100-NEXT: S_NOP 1 ; gfx908-PAD100-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-DEFAULT-LABEL: name: mfma_padding_2_pass + ; gfx90a-DEFAULT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-DEFAULT-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD50-LABEL: name: mfma_padding_2_pass + ; gfx90a-PAD50: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD50-NEXT: S_NOP 0 + ; gfx90a-PAD50-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD100-LABEL: name: mfma_padding_2_pass + ; gfx90a-PAD100: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD100-NEXT: S_NOP 1 + ; gfx90a-PAD100-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-DEFAULT-LABEL: name: mfma_padding_2_pass + ; gfx940-DEFAULT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-DEFAULT-NEXT: S_NOP 1 + ; gfx940-DEFAULT-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD50-LABEL: name: mfma_padding_2_pass + ; gfx940-PAD50: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD50-NEXT: S_NOP 1 + ; gfx940-PAD50-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD100-LABEL: name: mfma_padding_2_pass + ; gfx940-PAD100: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD100-NEXT: S_NOP 1 + ; gfx940-PAD100-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec ... @@ -64,6 +101,40 @@ body: | ; gfx908-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec ; gfx908-PAD100-NEXT: S_NOP 0 ; gfx908-PAD100-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-DEFAULT-LABEL: name: mfma_padding_2_pass_1_intervening_valu + ; gfx90a-DEFAULT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD50-LABEL: name: mfma_padding_2_pass_1_intervening_valu + ; gfx90a-PAD50: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD100-LABEL: name: mfma_padding_2_pass_1_intervening_valu + ; gfx90a-PAD100: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: S_NOP 0 + ; gfx90a-PAD100-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-DEFAULT-LABEL: name: mfma_padding_2_pass_1_intervening_valu + ; gfx940-DEFAULT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: S_NOP 0 + ; gfx940-DEFAULT-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD50-LABEL: name: mfma_padding_2_pass_1_intervening_valu + ; gfx940-PAD50: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: S_NOP 0 + ; gfx940-PAD50-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD100-LABEL: name: mfma_padding_2_pass_1_intervening_valu + ; gfx940-PAD100: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: S_NOP 0 + ; gfx940-PAD100-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec $vgpr2 = V_MOV_B32_e32 1, implicit $exec $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec @@ -100,6 +171,41 @@ body: | ; gfx908-PAD100-NEXT: DBG_VALUE ; gfx908-PAD100-NEXT: S_NOP 1 ; gfx908-PAD100-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-DEFAULT-LABEL: name: mfma_padding_2_pass_dbg + ; gfx90a-DEFAULT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-DEFAULT-NEXT: DBG_VALUE + ; gfx90a-DEFAULT-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD50-LABEL: name: mfma_padding_2_pass_dbg + ; gfx90a-PAD50: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD50-NEXT: DBG_VALUE + ; gfx90a-PAD50-NEXT: S_NOP 0 + ; gfx90a-PAD50-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD100-LABEL: name: mfma_padding_2_pass_dbg + ; gfx90a-PAD100: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD100-NEXT: DBG_VALUE + ; gfx90a-PAD100-NEXT: S_NOP 1 + ; gfx90a-PAD100-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-DEFAULT-LABEL: name: mfma_padding_2_pass_dbg + ; gfx940-DEFAULT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-DEFAULT-NEXT: DBG_VALUE + ; gfx940-DEFAULT-NEXT: S_NOP 1 + ; gfx940-DEFAULT-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD50-LABEL: name: mfma_padding_2_pass_dbg + ; gfx940-PAD50: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD50-NEXT: DBG_VALUE + ; gfx940-PAD50-NEXT: S_NOP 1 + ; gfx940-PAD50-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD100-LABEL: name: mfma_padding_2_pass_dbg + ; gfx940-PAD100: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD100-NEXT: DBG_VALUE + ; gfx940-PAD100-NEXT: S_NOP 1 + ; gfx940-PAD100-NEXT: $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec DBG_VALUE $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec @@ -132,6 +238,34 @@ body: | ; gfx908-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec ; gfx908-PAD100-NEXT: S_NOP 7 ; gfx908-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-DEFAULT-LABEL: name: mfma_padding_8_pass + ; gfx90a-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD50-LABEL: name: mfma_padding_8_pass + ; gfx90a-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD50-NEXT: S_NOP 3 + ; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD100-LABEL: name: mfma_padding_8_pass + ; gfx90a-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD100-NEXT: S_NOP 7 + ; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-DEFAULT-LABEL: name: mfma_padding_8_pass + ; gfx940-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD50-LABEL: name: mfma_padding_8_pass + ; gfx940-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD50-NEXT: S_NOP 3 + ; gfx940-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD100-LABEL: name: mfma_padding_8_pass + ; gfx940-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD100-NEXT: S_NOP 7 + ; gfx940-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec ... @@ -172,6 +306,46 @@ body: | ; gfx908-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec ; gfx908-PAD100-NEXT: S_NOP 5 ; gfx908-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-DEFAULT-LABEL: name: mfma_padding_8_pass_2_intervening_valu + ; gfx90a-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD50-LABEL: name: mfma_padding_8_pass_2_intervening_valu + ; gfx90a-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: S_NOP 1 + ; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD100-LABEL: name: mfma_padding_8_pass_2_intervening_valu + ; gfx90a-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: S_NOP 5 + ; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-DEFAULT-LABEL: name: mfma_padding_8_pass_2_intervening_valu + ; gfx940-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD50-LABEL: name: mfma_padding_8_pass_2_intervening_valu + ; gfx940-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: S_NOP 1 + ; gfx940-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD100-LABEL: name: mfma_padding_8_pass_2_intervening_valu + ; gfx940-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: S_NOP 5 + ; gfx940-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $vgpr2 = V_MOV_B32_e32 1, implicit $exec $vgpr3 = V_MOV_B32_e32 1, implicit $exec @@ -207,6 +381,36 @@ body: | ; gfx908-PAD100-NEXT: S_NOP 7 ; gfx908-PAD100-NEXT: S_NOP 7 ; gfx908-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-DEFAULT-LABEL: name: mfma_padding_16_pass + ; gfx90a-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD50-LABEL: name: mfma_padding_16_pass + ; gfx90a-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD50-NEXT: S_NOP 7 + ; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD100-LABEL: name: mfma_padding_16_pass + ; gfx90a-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD100-NEXT: S_NOP 7 + ; gfx90a-PAD100-NEXT: S_NOP 7 + ; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-DEFAULT-LABEL: name: mfma_padding_16_pass + ; gfx940-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD50-LABEL: name: mfma_padding_16_pass + ; gfx940-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD50-NEXT: S_NOP 7 + ; gfx940-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD100-LABEL: name: mfma_padding_16_pass + ; gfx940-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD100-NEXT: S_NOP 7 + ; gfx940-PAD100-NEXT: S_NOP 7 + ; gfx940-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec ... @@ -258,6 +462,60 @@ body: | ; gfx908-PAD100-NEXT: S_NOP 7 ; gfx908-PAD100-NEXT: S_NOP 3 ; gfx908-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-DEFAULT-LABEL: name: mfma_padding_16_pass_4_intervening_valu + ; gfx90a-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD50-LABEL: name: mfma_padding_16_pass_4_intervening_valu + ; gfx90a-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: S_NOP 3 + ; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD100-LABEL: name: mfma_padding_16_pass_4_intervening_valu + ; gfx90a-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: S_NOP 7 + ; gfx90a-PAD100-NEXT: S_NOP 3 + ; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-DEFAULT-LABEL: name: mfma_padding_16_pass_4_intervening_valu + ; gfx940-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD50-LABEL: name: mfma_padding_16_pass_4_intervening_valu + ; gfx940-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: S_NOP 3 + ; gfx940-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD100-LABEL: name: mfma_padding_16_pass_4_intervening_valu + ; gfx940-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: S_NOP 7 + ; gfx940-PAD100-NEXT: S_NOP 3 + ; gfx940-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $vgpr2 = V_MOV_B32_e32 1, implicit $exec $vgpr3 = V_MOV_B32_e32 1, implicit $exec @@ -369,6 +627,126 @@ body: | ; gfx908-PAD100-NEXT: $vgpr16 = V_MOV_B32_e32 1, implicit $exec ; gfx908-PAD100-NEXT: $vgpr17 = V_MOV_B32_e32 1, implicit $exec ; gfx908-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-DEFAULT-LABEL: name: mfma_padding_16_pass_16_intervening_valu + ; gfx90a-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr6 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr7 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr8 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr9 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr10 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr11 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr12 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr13 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr14 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr15 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr16 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: $vgpr17 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD50-LABEL: name: mfma_padding_16_pass_16_intervening_valu + ; gfx90a-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr6 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr7 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr8 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr9 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr10 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr11 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr12 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr13 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr14 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr15 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr16 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: $vgpr17 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD100-LABEL: name: mfma_padding_16_pass_16_intervening_valu + ; gfx90a-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr6 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr7 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr8 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr9 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr10 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr11 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr12 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr13 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr14 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr15 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr16 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: $vgpr17 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-DEFAULT-LABEL: name: mfma_padding_16_pass_16_intervening_valu + ; gfx940-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr6 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr7 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr8 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr9 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr10 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr11 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr12 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr13 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr14 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr15 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr16 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: $vgpr17 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD50-LABEL: name: mfma_padding_16_pass_16_intervening_valu + ; gfx940-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr6 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr7 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr8 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr9 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr10 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr11 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr12 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr13 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr14 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr15 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr16 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: $vgpr17 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD100-LABEL: name: mfma_padding_16_pass_16_intervening_valu + ; gfx940-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr6 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr7 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr8 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr9 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr10 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr11 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr12 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr13 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr14 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr15 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr16 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: $vgpr17 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $vgpr2 = V_MOV_B32_e32 1, implicit $exec $vgpr3 = V_MOV_B32_e32 1, implicit $exec @@ -414,6 +792,30 @@ body: | ; gfx908-PAD100-LABEL: name: mfma_padding_16_pass_occ_1 ; gfx908-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec ; gfx908-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-DEFAULT-LABEL: name: mfma_padding_16_pass_occ_1 + ; gfx90a-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD50-LABEL: name: mfma_padding_16_pass_occ_1 + ; gfx90a-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD100-LABEL: name: mfma_padding_16_pass_occ_1 + ; gfx90a-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-DEFAULT-LABEL: name: mfma_padding_16_pass_occ_1 + ; gfx940-DEFAULT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD50-LABEL: name: mfma_padding_16_pass_occ_1 + ; gfx940-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD100-LABEL: name: mfma_padding_16_pass_occ_1 + ; gfx940-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec ... @@ -506,6 +908,108 @@ body: | ; gfx908-PAD100-NEXT: S_NOP 7 ; gfx908-PAD100-NEXT: S_NOP 5 ; gfx908-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-DEFAULT-LABEL: name: mfma_padding_16_pass_2_preds + ; gfx90a-DEFAULT: bb.0: + ; gfx90a-DEFAULT-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000) + ; gfx90a-DEFAULT-NEXT: {{ $}} + ; gfx90a-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-DEFAULT-NEXT: S_CBRANCH_VCCZ %bb.2, implicit undef $vcc + ; gfx90a-DEFAULT-NEXT: {{ $}} + ; gfx90a-DEFAULT-NEXT: bb.1: + ; gfx90a-DEFAULT-NEXT: successors: %bb.2(0x80000000) + ; gfx90a-DEFAULT-NEXT: {{ $}} + ; gfx90a-DEFAULT-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: {{ $}} + ; gfx90a-DEFAULT-NEXT: bb.2: + ; gfx90a-DEFAULT-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD50-LABEL: name: mfma_padding_16_pass_2_preds + ; gfx90a-PAD50: bb.0: + ; gfx90a-PAD50-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000) + ; gfx90a-PAD50-NEXT: {{ $}} + ; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD50-NEXT: S_CBRANCH_VCCZ %bb.2, implicit undef $vcc + ; gfx90a-PAD50-NEXT: {{ $}} + ; gfx90a-PAD50-NEXT: bb.1: + ; gfx90a-PAD50-NEXT: successors: %bb.2(0x80000000) + ; gfx90a-PAD50-NEXT: {{ $}} + ; gfx90a-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: {{ $}} + ; gfx90a-PAD50-NEXT: bb.2: + ; gfx90a-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD50-NEXT: S_NOP 5 + ; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx90a-PAD100-LABEL: name: mfma_padding_16_pass_2_preds + ; gfx90a-PAD100: bb.0: + ; gfx90a-PAD100-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000) + ; gfx90a-PAD100-NEXT: {{ $}} + ; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx90a-PAD100-NEXT: S_CBRANCH_VCCZ %bb.2, implicit undef $vcc + ; gfx90a-PAD100-NEXT: {{ $}} + ; gfx90a-PAD100-NEXT: bb.1: + ; gfx90a-PAD100-NEXT: successors: %bb.2(0x80000000) + ; gfx90a-PAD100-NEXT: {{ $}} + ; gfx90a-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: {{ $}} + ; gfx90a-PAD100-NEXT: bb.2: + ; gfx90a-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx90a-PAD100-NEXT: S_NOP 7 + ; gfx90a-PAD100-NEXT: S_NOP 5 + ; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-DEFAULT-LABEL: name: mfma_padding_16_pass_2_preds + ; gfx940-DEFAULT: bb.0: + ; gfx940-DEFAULT-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000) + ; gfx940-DEFAULT-NEXT: {{ $}} + ; gfx940-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-DEFAULT-NEXT: S_CBRANCH_VCCZ %bb.2, implicit undef $vcc + ; gfx940-DEFAULT-NEXT: {{ $}} + ; gfx940-DEFAULT-NEXT: bb.1: + ; gfx940-DEFAULT-NEXT: successors: %bb.2(0x80000000) + ; gfx940-DEFAULT-NEXT: {{ $}} + ; gfx940-DEFAULT-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: {{ $}} + ; gfx940-DEFAULT-NEXT: bb.2: + ; gfx940-DEFAULT-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-DEFAULT-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD50-LABEL: name: mfma_padding_16_pass_2_preds + ; gfx940-PAD50: bb.0: + ; gfx940-PAD50-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000) + ; gfx940-PAD50-NEXT: {{ $}} + ; gfx940-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD50-NEXT: S_CBRANCH_VCCZ %bb.2, implicit undef $vcc + ; gfx940-PAD50-NEXT: {{ $}} + ; gfx940-PAD50-NEXT: bb.1: + ; gfx940-PAD50-NEXT: successors: %bb.2(0x80000000) + ; gfx940-PAD50-NEXT: {{ $}} + ; gfx940-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: {{ $}} + ; gfx940-PAD50-NEXT: bb.2: + ; gfx940-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD50-NEXT: S_NOP 5 + ; gfx940-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; + ; gfx940-PAD100-LABEL: name: mfma_padding_16_pass_2_preds + ; gfx940-PAD100: bb.0: + ; gfx940-PAD100-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000) + ; gfx940-PAD100-NEXT: {{ $}} + ; gfx940-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + ; gfx940-PAD100-NEXT: S_CBRANCH_VCCZ %bb.2, implicit undef $vcc + ; gfx940-PAD100-NEXT: {{ $}} + ; gfx940-PAD100-NEXT: bb.1: + ; gfx940-PAD100-NEXT: successors: %bb.2(0x80000000) + ; gfx940-PAD100-NEXT: {{ $}} + ; gfx940-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: {{ $}} + ; gfx940-PAD100-NEXT: bb.2: + ; gfx940-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec + ; gfx940-PAD100-NEXT: S_NOP 7 + ; gfx940-PAD100-NEXT: S_NOP 5 + ; gfx940-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec bb.0: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec S_CBRANCH_VCCZ %bb.2, implicit undef $vcc -- GitLab From b5b34dbb27359139ef1eb2ca22e8c5a954e34e50 Mon Sep 17 00:00:00 2001 From: Austin Kerbow Date: Sun, 31 Mar 2024 11:03:03 -0700 Subject: [PATCH 0068/1690] [AMDGPU] Use directive for kernarg preload header padding (#86004) --- .../MCTargetDesc/AMDGPUTargetStreamer.cpp | 21 +- .../CodeGen/AMDGPU/preload-kernarg-header.ll | 11 +- llvm/test/CodeGen/AMDGPU/preload-kernargs.ll | 9050 +---------------- 3 files changed, 325 insertions(+), 8757 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp index 3006fcdb9282..02fe7be06280 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp @@ -284,6 +284,15 @@ bool AMDGPUTargetAsmStreamer::EmitHSAMetadata( return true; } +bool AMDGPUTargetAsmStreamer::EmitKernargPreloadHeader( + const MCSubtargetInfo &STI, bool TrapEnabled) { + OS << (TrapEnabled ? "\ts_trap 2" : "\ts_endpgm") + << " ; Kernarg preload header. Trap with incompatible firmware that " + "doesn't support preloading kernel arguments.\n"; + OS << "\t.fill 63, 4, 0xbf800000 ; s_nop 0\n"; + return true; +} + bool AMDGPUTargetAsmStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) { const uint32_t Encoded_s_code_end = 0xbf9f0000; const uint32_t Encoded_s_nop = 0xbf800000; @@ -861,18 +870,6 @@ bool AMDGPUTargetELFStreamer::EmitHSAMetadata(msgpack::Document &HSAMetadataDoc, return true; } -bool AMDGPUTargetAsmStreamer::EmitKernargPreloadHeader( - const MCSubtargetInfo &STI, bool TrapEnabled) { - const char *TrapInstr = TrapEnabled ? "\ts_trap 2" : "\ts_endpgm"; - OS << TrapInstr - << " ; Trap with incompatible firmware that doesn't " - "support preloading kernel arguments.\n"; - for (int i = 0; i < 63; ++i) { - OS << "\ts_nop 0\n"; - } - return true; -} - bool AMDGPUTargetELFStreamer::EmitKernargPreloadHeader( const MCSubtargetInfo &STI, bool TrapEnabled) { const uint32_t Encoded_s_nop = 0xbf800000; diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernarg-header.ll b/llvm/test/CodeGen/AMDGPU/preload-kernarg-header.ll index a70488a00db7..a030f86da1b6 100644 --- a/llvm/test/CodeGen/AMDGPU/preload-kernarg-header.ll +++ b/llvm/test/CodeGen/AMDGPU/preload-kernarg-header.ll @@ -1,17 +1,20 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -asm-verbose=0 < %s | FileCheck -check-prefixes=GCN,HSA %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -filetype=obj < %s | llvm-objdump --arch=amdgcn --mcpu=gfx940 --disassemble - | FileCheck -check-prefixes=GCN,HSA %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -filetype=obj < %s | llvm-objdump --arch=amdgcn --mcpu=gfx940 --disassemble - | FileCheck -check-prefixes=GCN,NON-HSA %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -asm-verbose=0 < %s | FileCheck -check-prefixes=GCN,HSA,ASM %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -filetype=obj < %s | llvm-objdump --arch=amdgcn --mcpu=gfx940 --disassemble - | FileCheck -check-prefixes=GCN,HSA,OBJ %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -filetype=obj < %s | llvm-objdump --arch=amdgcn --mcpu=gfx940 --disassemble - | FileCheck -check-prefixes=GCN,NON-HSA,OBJ %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -asm-verbose=0 < %s | llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx940 -filetype=obj | llvm-objdump --arch=amdgcn --mcpu=gfx940 --disassemble - | FileCheck -check-prefixes=GCN,HSA,OBJ %s ; GCN: preload_kernarg_header ; HSA: s_trap 2 ; NON-HSA: s_endpgm -; GCN-COUNT-63: s_nop 0 +; ASM: .fill 63, 4, 0xbf800000 ; s_nop 0 +; OBJ-COUNT-63: s_nop 0 define amdgpu_kernel void @preload_kernarg_header(ptr %arg) { store ptr %arg, ptr %arg ret void } ; GCN: non_kernel_function +; GCN-NOT: s_trap 2 ; GCN-NOT: s_nop 0 ; GCN: flat_store define void @non_kernel_function(ptr %arg) { diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll index d20c3a4007ff..f0e709b5a172 100644 --- a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll +++ b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll @@ -24,70 +24,8 @@ define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) { ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: ptr1_i8: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -98,70 +36,8 @@ define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) { ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: ptr1_i8: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: s_and_b32 s0, s4, 0xff ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 @@ -170,70 +46,8 @@ define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) { ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: ptr1_i8: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: s_and_b32 s0, s4, 0xff ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 @@ -242,70 +56,8 @@ define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) { ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: ptr1_i8: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: s_and_b32 s0, s4, 0xff ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 @@ -325,70 +77,8 @@ define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) { ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: ptr1_i8: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -399,70 +89,8 @@ define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) { ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: ptr1_i8: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: s_and_b32 s0, s8, 0xff ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 @@ -471,70 +99,8 @@ define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) { ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: ptr1_i8: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: s_and_b32 s0, s8, 0xff ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 @@ -543,70 +109,8 @@ define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) { ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: ptr1_i8: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: s_and_b32 s0, s8, 0xff ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 @@ -631,70 +135,8 @@ define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %a ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: ptr1_i8_zext_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -705,70 +147,8 @@ define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %a ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: ptr1_i8_zext_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: s_mov_b32 s0, 0xffff ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s4 @@ -778,70 +158,8 @@ define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %a ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: ptr1_i8_zext_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: s_mov_b32 s0, 0xffff ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s4 @@ -851,70 +169,8 @@ define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %a ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: ptr1_i8_zext_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: s_mov_b32 s0, 0xffff ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s4 @@ -935,70 +191,8 @@ define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %a ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: ptr1_i8_zext_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -1009,70 +203,8 @@ define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %a ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: ptr1_i8_zext_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: s_mov_b32 s0, 0xffff ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s8 @@ -1082,70 +214,8 @@ define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %a ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: ptr1_i8_zext_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: s_mov_b32 s0, 0xffff ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s8 @@ -1155,70 +225,8 @@ define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %a ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: ptr1_i8_zext_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: s_mov_b32 s0, 0xffff ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s8 @@ -1244,70 +252,8 @@ define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0 ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: ptr1_i16_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -1318,70 +264,8 @@ define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0 ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: ptr1_i16_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: s_and_b32 s0, s4, 0xffff ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 @@ -1390,70 +274,8 @@ define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0 ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: ptr1_i16_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: s_and_b32 s0, s4, 0xffff ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 @@ -1462,70 +284,8 @@ define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0 ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: ptr1_i16_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: s_and_b32 s0, s4, 0xffff ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 @@ -1545,70 +305,8 @@ define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0 ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: ptr1_i16_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -1619,70 +317,8 @@ define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0 ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: ptr1_i16_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: s_and_b32 s0, s8, 0xffff ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 @@ -1691,70 +327,8 @@ define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0 ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: ptr1_i16_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: s_and_b32 s0, s8, 0xffff ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 @@ -1763,70 +337,8 @@ define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0 ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: ptr1_i16_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: s_and_b32 s0, s8, 0xffff ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 @@ -1850,70 +362,8 @@ define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0 ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: ptr1_i32_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -1923,70 +373,8 @@ define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0 ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: ptr1_i32_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s4 @@ -1994,70 +382,8 @@ define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0 ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: ptr1_i32_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s4 @@ -2065,70 +391,8 @@ define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0 ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: ptr1_i32_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s4 @@ -2146,70 +410,8 @@ define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0 ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: ptr1_i32_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -2219,70 +421,8 @@ define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0 ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: ptr1_i32_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s8 @@ -2290,70 +430,8 @@ define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0 ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: ptr1_i32_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s8 @@ -2361,70 +439,8 @@ define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0 ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: ptr1_i32_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s8 @@ -2449,70 +465,8 @@ define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: i32_ptr1_i32_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dword s3, s[0:1], 0x10 ; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 @@ -2524,70 +478,8 @@ define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: i32_ptr1_i32_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: s_load_dword s0, s[0:1], 0x10 ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 @@ -2598,70 +490,8 @@ define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: i32_ptr1_i32_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: s_add_i32 s0, s2, s6 ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 @@ -2670,70 +500,8 @@ define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: i32_ptr1_i32_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: s_add_i32 s0, s2, s6 ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 @@ -2754,70 +522,8 @@ define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: i32_ptr1_i32_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dword s2, s[4:5], 0x10 ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 @@ -2829,70 +535,8 @@ define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: i32_ptr1_i32_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: s_load_dword s0, s[4:5], 0x10 ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 @@ -2903,70 +547,8 @@ define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: i32_ptr1_i32_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: s_add_i32 s0, s6, s10 ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 @@ -2975,70 +557,8 @@ define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: i32_ptr1_i32_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: s_add_i32 s0, s6, s10 ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 @@ -3065,70 +585,8 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 % ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: ptr1_i16_i16_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -3141,70 +599,8 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 % ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: ptr1_i16_i16_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: s_load_dword s0, s[0:1], 0x8 ; GFX940-PRELOAD-2-NEXT: s_and_b32 s1, s4, 0xffff @@ -3217,70 +613,8 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 % ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: ptr1_i16_i16_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 16 ; GFX940-PRELOAD-4-NEXT: s_and_b32 s1, s4, 0xffff @@ -3291,70 +625,8 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 % ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: ptr1_i16_i16_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 16 ; GFX940-PRELOAD-8-NEXT: s_and_b32 s1, s4, 0xffff @@ -3378,70 +650,8 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 % ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: ptr1_i16_i16_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -3454,70 +664,8 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 % ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: ptr1_i16_i16_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: s_load_dword s0, s[4:5], 0x8 ; GFX90a-PRELOAD-2-NEXT: s_and_b32 s1, s8, 0xffff @@ -3530,70 +678,8 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 % ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: ptr1_i16_i16_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s8, 16 ; GFX90a-PRELOAD-4-NEXT: s_and_b32 s1, s8, 0xffff @@ -3604,70 +690,8 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 % ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: ptr1_i16_i16_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s8, 16 ; GFX90a-PRELOAD-8-NEXT: s_and_b32 s1, s8, 0xffff @@ -3695,70 +719,8 @@ define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: ptr1_v2i8_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -3768,70 +730,8 @@ define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: ptr1_v2i8_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 8 ; GFX940-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -3841,70 +741,8 @@ define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: ptr1_v2i8_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 8 ; GFX940-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -3914,70 +752,8 @@ define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: ptr1_v2i8_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 8 ; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -3997,70 +773,8 @@ define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: ptr1_v2i8_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -4070,70 +784,8 @@ define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: ptr1_v2i8_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s8, 8 ; GFX90a-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -4143,70 +795,8 @@ define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: ptr1_v2i8_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s8, 8 ; GFX90a-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -4216,70 +806,8 @@ define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: ptr1_v2i8_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s8, 8 ; GFX90a-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -4308,70 +836,8 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspac ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: byref_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -4385,70 +851,8 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspac ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: byref_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 @@ -4462,70 +866,8 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspac ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: byref_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 @@ -4539,70 +881,8 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspac ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: byref_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 @@ -4630,70 +910,8 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspac ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: byref_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x100 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -4707,70 +925,8 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspac ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: byref_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x100 ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 @@ -4784,70 +940,8 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspac ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: byref_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x100 ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 @@ -4861,70 +955,8 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspac ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: byref_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x100 ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 @@ -4964,70 +996,8 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: v8i32_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v4, 0 @@ -5046,70 +1016,8 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: v8i32_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v4, 0 @@ -5128,70 +1036,8 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: v8i32_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v4, 0 @@ -5210,70 +1056,8 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: v8i32_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v4, 0 @@ -5311,70 +1095,8 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: v8i32_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x20 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v4, 0 @@ -5393,79 +1115,17 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: v8i32_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: ; %bb.0: -; GFX90a-PRELOAD-2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x20 -; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v4, 0 -; GFX90a-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) -; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s12 -; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s13 -; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s14 -; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v3, s15 -; GFX90a-PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x20 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 ; GFX90a-PRELOAD-2-NEXT: s_nop 0 ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s8 ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s9 @@ -5475,70 +1135,8 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: v8i32_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x20 ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v4, 0 @@ -5557,70 +1155,8 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: v8i32_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x20 ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v4, 0 @@ -5654,70 +1190,8 @@ define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: v3i16_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -5729,70 +1203,8 @@ define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: v3i16_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s5 @@ -5802,70 +1214,8 @@ define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: v3i16_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s5 @@ -5875,70 +1225,8 @@ define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: v3i16_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s5 @@ -5959,70 +1247,8 @@ define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: v3i16_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -6034,70 +1260,8 @@ define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: v3i16_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s9 @@ -6107,70 +1271,8 @@ define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: v3i16_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s9 @@ -6180,70 +1282,8 @@ define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: v3i16_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s9 @@ -6269,70 +1309,8 @@ define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: v3i32_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x10 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v3, 0 @@ -6344,70 +1322,8 @@ define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: v3i32_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s6 ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s7 @@ -6417,70 +1333,8 @@ define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: v3i32_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s6 ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s7 @@ -6490,70 +1344,8 @@ define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: v3i32_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s6 ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s7 @@ -6575,70 +1367,8 @@ define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: v3i32_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v3, 0 @@ -6650,70 +1380,8 @@ define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: v3i32_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s10 ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s11 @@ -6723,70 +1391,8 @@ define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: v3i32_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s10 ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s11 @@ -6796,70 +1402,8 @@ define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: v3i32_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s10 ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s11 @@ -6885,70 +1429,8 @@ define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: v3f32_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x10 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v3, 0 @@ -6960,70 +1442,8 @@ define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: v3f32_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v3, 0 ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s6 @@ -7033,70 +1453,8 @@ define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: v3f32_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v3, 0 ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s6 @@ -7106,70 +1464,8 @@ define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: v3f32_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v3, 0 ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s6 @@ -7191,70 +1487,8 @@ define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: v3f32_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v3, 0 @@ -7266,70 +1500,8 @@ define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: v3f32_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v3, 0 ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s10 @@ -7339,70 +1511,8 @@ define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: v3f32_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v3, 0 ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s10 @@ -7412,70 +1522,8 @@ define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: v3f32_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v3, 0 ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s10 @@ -7500,70 +1548,8 @@ define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: v5i8_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -7575,70 +1561,8 @@ define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: v5i8_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 8 ; GFX940-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -7655,70 +1579,8 @@ define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: v5i8_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 8 ; GFX940-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -7735,70 +1597,8 @@ define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: v5i8_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 8 ; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -7826,70 +1626,8 @@ define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: v5i8_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 @@ -7901,70 +1639,8 @@ define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: v5i8_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s8, 8 ; GFX90a-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -7981,70 +1657,8 @@ define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: v5i8_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s8, 8 ; GFX90a-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -8061,70 +1675,8 @@ define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: v5i8_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s8, 8 ; GFX90a-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -8167,70 +1719,8 @@ define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x doubl ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: v5f64_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 ; GFX940-PRELOAD-1-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 @@ -8252,70 +1742,8 @@ define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x doubl ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: v5f64_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 ; GFX940-PRELOAD-2-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 @@ -8337,70 +1765,8 @@ define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x doubl ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: v5f64_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 ; GFX940-PRELOAD-4-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 @@ -8422,70 +1788,8 @@ define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x doubl ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: v5f64_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 ; GFX940-PRELOAD-8-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 @@ -8529,70 +1833,8 @@ define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x doubl ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: v5f64_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x60 ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40 @@ -8614,70 +1856,8 @@ define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x doubl ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: v5f64_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x60 ; GFX90a-PRELOAD-2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40 @@ -8699,70 +1879,8 @@ define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x doubl ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: v5f64_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x60 ; GFX90a-PRELOAD-4-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40 @@ -8784,82 +1902,20 @@ define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x doubl ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: v5f64_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: ; %bb.0: -; GFX90a-PRELOAD-8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x60 -; GFX90a-PRELOAD-8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40 -; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v4, 0 -; GFX90a-PRELOAD-8-NEXT: s_waitcnt lgkmcnt(0) -; GFX90a-PRELOAD-8-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1] -; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s12 -; GFX90a-PRELOAD-8-NEXT: global_store_dwordx2 v4, v[2:3], s[6:7] offset:32 -; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s13 -; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s14 -; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v3, s15 -; GFX90a-PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x60 +; GFX90a-PRELOAD-8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-PRELOAD-8-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-8-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1] +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx2 v4, v[2:3], s[6:7] offset:32 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 ; GFX90a-PRELOAD-8-NEXT: s_nop 0 ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s8 ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s9 @@ -8882,70 +1938,8 @@ define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: v8i8_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 @@ -8955,70 +1949,8 @@ define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: v8i8_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s5, 8 ; GFX940-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -9042,70 +1974,8 @@ define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: v8i8_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s5, 8 ; GFX940-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -9129,86 +1999,24 @@ define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: v8i8_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: ; %bb.0: -; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 8 -; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 24 -; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v1, 8, s0 -; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 16 -; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 8 -; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 24 -; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v2, 8, s0 -; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 16 -; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 8 +; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 24 +; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 16 +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 8 +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 24 +; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v2, 8, s0 +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 16 +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX940-PRELOAD-8-NEXT: s_nop 0 ; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 @@ -9225,70 +2033,8 @@ define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: v8i8_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 @@ -9298,70 +2044,8 @@ define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: v8i8_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s9, 8 ; GFX90a-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -9384,70 +2068,8 @@ define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: v8i8_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s9, 8 ; GFX90a-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -9470,70 +2092,8 @@ define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: v8i8_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s9, 8 ; GFX90a-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 @@ -9570,70 +2130,8 @@ define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: i64_kernel_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 @@ -9643,70 +2141,8 @@ define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: i64_kernel_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-PRELOAD-2-NEXT: v_mov_b64_e32 v[0:1], s[4:5] @@ -9714,70 +2150,8 @@ define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: i64_kernel_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-PRELOAD-4-NEXT: v_mov_b64_e32 v[0:1], s[4:5] @@ -9785,70 +2159,8 @@ define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: i64_kernel_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-PRELOAD-8-NEXT: v_mov_b64_e32 v[0:1], s[4:5] @@ -9866,70 +2178,8 @@ define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: i64_kernel_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 @@ -9939,70 +2189,8 @@ define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: i64_kernel_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 ; GFX90a-PRELOAD-2-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] @@ -10010,70 +2198,8 @@ define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: i64_kernel_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 ; GFX90a-PRELOAD-4-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] @@ -10081,70 +2207,8 @@ define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: i64_kernel_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 ; GFX90a-PRELOAD-8-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] @@ -10166,70 +2230,8 @@ define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double ; GFX940-NO-PRELOAD-NEXT: s_endpgm ; ; GFX940-PRELOAD-1-LABEL: f64_kernel_preload_arg: -; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 -; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-1-NEXT: ; %bb.0: ; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 ; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 @@ -10239,70 +2241,8 @@ define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double ; GFX940-PRELOAD-1-NEXT: s_endpgm ; ; GFX940-PRELOAD-2-LABEL: f64_kernel_preload_arg: -; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 -; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-2-NEXT: ; %bb.0: ; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-PRELOAD-2-NEXT: v_mov_b64_e32 v[0:1], s[4:5] @@ -10310,70 +2250,8 @@ define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double ; GFX940-PRELOAD-2-NEXT: s_endpgm ; ; GFX940-PRELOAD-4-LABEL: f64_kernel_preload_arg: -; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 -; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-4-NEXT: ; %bb.0: ; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-PRELOAD-4-NEXT: v_mov_b64_e32 v[0:1], s[4:5] @@ -10381,70 +2259,8 @@ define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double ; GFX940-PRELOAD-4-NEXT: s_endpgm ; ; GFX940-PRELOAD-8-LABEL: f64_kernel_preload_arg: -; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 -; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX940-PRELOAD-8-NEXT: ; %bb.0: ; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-PRELOAD-8-NEXT: v_mov_b64_e32 v[0:1], s[4:5] @@ -10462,70 +2278,8 @@ define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double ; GFX90a-NO-PRELOAD-NEXT: s_endpgm ; ; GFX90a-PRELOAD-1-LABEL: f64_kernel_preload_arg: -; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 -; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-1-NEXT: ; %bb.0: ; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 ; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 @@ -10535,70 +2289,8 @@ define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double ; GFX90a-PRELOAD-1-NEXT: s_endpgm ; ; GFX90a-PRELOAD-2-LABEL: f64_kernel_preload_arg: -; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 -; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-2-NEXT: ; %bb.0: ; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 ; GFX90a-PRELOAD-2-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] @@ -10606,70 +2298,8 @@ define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double ; GFX90a-PRELOAD-2-NEXT: s_endpgm ; ; GFX90a-PRELOAD-4-LABEL: f64_kernel_preload_arg: -; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 -; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-4-NEXT: ; %bb.0: ; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 ; GFX90a-PRELOAD-4-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] @@ -10677,70 +2307,8 @@ define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double ; GFX90a-PRELOAD-4-NEXT: s_endpgm ; ; GFX90a-PRELOAD-8-LABEL: f64_kernel_preload_arg: -; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 -; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8: s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: .fill 63, 4, 0xbf800000 ; s_nop 0 ; GFX90a-PRELOAD-8-NEXT: ; %bb.0: ; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 ; GFX90a-PRELOAD-8-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] -- GitLab From d2b63ed0ca5c7eb3f8fc029e3a5705cf0ddab438 Mon Sep 17 00:00:00 2001 From: Jon Chesterfield Date: Sun, 31 Mar 2024 19:49:02 +0100 Subject: [PATCH 0069/1690] Revert "Revert "[compiler-rt] Allow building builtins.a without a libc (#86737)"" This reverts commit a2982a29fdfcfe2904754815c85f630a4dc6d88c. Buildbot failure that motivated the revert looks unrelated. --- compiler-rt/cmake/config-ix.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake index 46a6fdf8728f..911f48fa1381 100644 --- a/compiler-rt/cmake/config-ix.cmake +++ b/compiler-rt/cmake/config-ix.cmake @@ -235,9 +235,9 @@ set(COMPILER_RT_SUPPORTED_ARCH) # Try to compile a very simple source file to ensure we can target the given # platform. We use the results of these tests to build only the various target # runtime libraries supported by our current compilers cross-compiling -# abilities. +# abilities. Avoids using libc as that may not be available yet. set(SIMPLE_SOURCE ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/simple.cc) -file(WRITE ${SIMPLE_SOURCE} "#include \n#include \nint main(void) { printf(\"hello, world\"); }\n") +file(WRITE ${SIMPLE_SOURCE} "int main(void) { return 0; }\n") # Detect whether the current target platform is 32-bit or 64-bit, and setup # the correct commandline flags needed to attempt to target 32-bit and 64-bit. -- GitLab From 2b0ab05c4a6c25049a30c56e6f0487db70abb3d5 Mon Sep 17 00:00:00 2001 From: Jakub Kuderski Date: Sun, 31 Mar 2024 14:55:11 -0400 Subject: [PATCH 0070/1690] [SLP][NFC] Simplify type checks with isa predicates (#87182) For more context on isa predicates, see: https://github.com/llvm/llvm-project/pull/83753. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 95 +++++++------------ 1 file changed, 33 insertions(+), 62 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 2875e71081d9..2bc0c5dcc606 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -458,8 +458,7 @@ static SmallBitVector isUndefVector(const Value *V, /// ShuffleVectorInst/getShuffleCost? static std::optional isFixedVectorShuffle(ArrayRef VL, SmallVectorImpl &Mask) { - const auto *It = - find_if(VL, [](Value *V) { return isa(V); }); + const auto *It = find_if(VL, IsaPred); if (It == VL.end()) return std::nullopt; auto *EI0 = cast(*It); @@ -4695,12 +4694,8 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) { // TODO: add analysis of other gather nodes with extractelement // instructions and other values/instructions, not only undefs. if ((TE.getOpcode() == Instruction::ExtractElement || - (all_of(TE.Scalars, - [](Value *V) { - return isa(V); - }) && - any_of(TE.Scalars, - [](Value *V) { return isa(V); }))) && + (all_of(TE.Scalars, IsaPred) && + any_of(TE.Scalars, IsaPred))) && all_of(TE.Scalars, [](Value *V) { auto *EE = dyn_cast(V); return !EE || isa(EE->getVectorOperandType()); @@ -4721,7 +4716,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) { // might be transformed. int Sz = TE.Scalars.size(); if (isSplat(TE.Scalars) && !allConstant(TE.Scalars) && - count_if(TE.Scalars, UndefValue::classof) == Sz - 1) { + count_if(TE.Scalars, IsaPred) == Sz - 1) { const auto *It = find_if(TE.Scalars, [](Value *V) { return !isConstant(V); }); if (It == TE.Scalars.begin()) @@ -6345,11 +6340,10 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, UserTreeIdx.UserTE->State == TreeEntry::ScatterVectorize && !(S.getOpcode() && allSameBlock(VL))) { assert(S.OpValue->getType()->isPointerTy() && - count_if(VL, [](Value *V) { return isa(V); }) >= - 2 && + count_if(VL, IsaPred) >= 2 && "Expected pointers only."); // Reset S to make it GetElementPtr kind of node. - const auto *It = find_if(VL, [](Value *V) { return isa(V); }); + const auto *It = find_if(VL, IsaPred); assert(It != VL.end() && "Expected at least one GEP."); S = getSameOpcode(*It, *TLI); } @@ -6893,17 +6887,12 @@ unsigned BoUpSLP::canMapToVector(Type *T) const { bool BoUpSLP::canReuseExtract(ArrayRef VL, Value *OpValue, SmallVectorImpl &CurrentOrder, bool ResizeAllowed) const { - const auto *It = find_if(VL, [](Value *V) { - return isa(V); - }); + const auto *It = find_if(VL, IsaPred); assert(It != VL.end() && "Expected at least one extract instruction."); auto *E0 = cast(*It); - assert(all_of(VL, - [](Value *V) { - return isa( - V); - }) && - "Invalid opcode"); + assert( + all_of(VL, IsaPred) && + "Invalid opcode"); // Check if all of the extracts come from the same vector and from the // correct offset. Value *Vec = E0->getOperand(0); @@ -7575,7 +7564,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { } InstructionCost getBuildVectorCost(ArrayRef VL, Value *Root) { - if ((!Root && allConstant(VL)) || all_of(VL, UndefValue::classof)) + if ((!Root && allConstant(VL)) || all_of(VL, IsaPred)) return TTI::TCC_Free; auto *VecTy = FixedVectorType::get(VL.front()->getType(), VL.size()); InstructionCost GatherCost = 0; @@ -7743,13 +7732,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { } else if (!Root && isSplat(VL)) { // Found the broadcasting of the single scalar, calculate the cost as // the broadcast. - const auto *It = - find_if(VL, [](Value *V) { return !isa(V); }); + const auto *It = find_if_not(VL, IsaPred); assert(It != VL.end() && "Expected at least one non-undef value."); // Add broadcast for non-identity shuffle only. bool NeedShuffle = count(VL, *It) > 1 && - (VL.front() != *It || !all_of(VL.drop_front(), UndefValue::classof)); + (VL.front() != *It || !all_of(VL.drop_front(), IsaPred)); if (!NeedShuffle) return TTI.getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, std::distance(VL.begin(), It), @@ -7757,7 +7745,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { SmallVector ShuffleMask(VL.size(), PoisonMaskElem); transform(VL, ShuffleMask.begin(), [](Value *V) { - return isa(V) ? PoisonMaskElem : 0; + return isa(V) ? PoisonMaskElem : 0; }); InstructionCost InsertCost = TTI.getVectorInstrCost( Instruction::InsertElement, VecTy, CostKind, 0, @@ -7768,7 +7756,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { /*SubTp=*/nullptr, /*Args=*/*It); } return GatherCost + - (all_of(Gathers, UndefValue::classof) + (all_of(Gathers, IsaPred) ? TTI::TCC_Free : R.getGatherCost(Gathers, !Root && VL.equals(Gathers))); }; @@ -8178,9 +8166,8 @@ public: // Take credit for instruction that will become dead. if (EE->hasOneUse() || !PrevNodeFound) { Instruction *Ext = EE->user_back(); - if (isa(Ext) && all_of(Ext->users(), [](User *U) { - return isa(U); - })) { + if (isa(Ext) && + all_of(Ext->users(), IsaPred)) { // Use getExtractWithExtendCost() to calculate the cost of // extractelement/ext pair. Cost -= @@ -8645,8 +8632,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, if (I->hasOneUse()) { Instruction *Ext = I->user_back(); if ((isa(Ext) || isa(Ext)) && - all_of(Ext->users(), - [](User *U) { return isa(U); })) { + all_of(Ext->users(), IsaPred)) { // Use getExtractWithExtendCost() to calculate the cost of // extractelement/ext pair. InstructionCost Cost = TTI->getExtractWithExtendCost( @@ -9130,10 +9116,7 @@ bool BoUpSLP::isFullyVectorizableTinyTree(bool ForReduction) const { (allConstant(TE->Scalars) || isSplat(TE->Scalars) || TE->Scalars.size() < Limit || ((TE->getOpcode() == Instruction::ExtractElement || - all_of(TE->Scalars, - [](Value *V) { - return isa(V); - })) && + all_of(TE->Scalars, IsaPred)) && isFixedVectorShuffle(TE->Scalars, Mask)) || (TE->State == TreeEntry::NeedToGather && TE->getOpcode() == Instruction::Load && !TE->isAltShuffle())); @@ -9254,9 +9237,7 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const { all_of(VectorizableTree, [&](const std::unique_ptr &TE) { return (TE->State == TreeEntry::NeedToGather && TE->getOpcode() != Instruction::ExtractElement && - count_if(TE->Scalars, - [](Value *V) { return isa(V); }) <= - Limit) || + count_if(TE->Scalars, IsaPred) <= Limit) || TE->getOpcode() == Instruction::PHI; })) return true; @@ -9285,9 +9266,7 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const { return isa(V) || (IsAllowedSingleBVNode && !V->hasNUsesOrMore(UsesLimit) && - any_of(V->users(), [](User *U) { - return isa(U); - })); + any_of(V->users(), IsaPred)); }); })) return false; @@ -10284,7 +10263,7 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry( } } - bool IsSplatOrUndefs = isSplat(VL) || all_of(VL, UndefValue::classof); + bool IsSplatOrUndefs = isSplat(VL) || all_of(VL, IsaPred); // Checks if the 2 PHIs are compatible in terms of high possibility to be // vectorized. auto AreCompatiblePHIs = [&](Value *V, Value *V1) { @@ -11261,8 +11240,7 @@ Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx, InstructionsState S = getSameOpcode(VL, *TLI); // Special processing for GEPs bundle, which may include non-gep values. if (!S.getOpcode() && VL.front()->getType()->isPointerTy()) { - const auto *It = - find_if(VL, [](Value *V) { return isa(V); }); + const auto *It = find_if(VL, IsaPred); if (It != VL.end()) S = getSameOpcode(*It, *TLI); } @@ -11432,7 +11410,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) { unsigned NumParts = TTI->getNumberOfParts(VecTy); if (NumParts == 0 || NumParts >= GatheredScalars.size()) NumParts = 1; - if (!all_of(GatheredScalars, UndefValue::classof)) { + if (!all_of(GatheredScalars, IsaPred)) { // Check for gathered extracts. bool Resized = false; ExtractShuffles = @@ -11757,7 +11735,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) { GatheredScalars[I] = PoisonValue::get(ScalarTy); } // Generate constants for final shuffle and build a mask for them. - if (!all_of(GatheredScalars, PoisonValue::classof)) { + if (!all_of(GatheredScalars, IsaPred)) { SmallVector BVMask(GatheredScalars.size(), PoisonMaskElem); TryPackScalars(GatheredScalars, BVMask, /*IsRootPoison=*/true); Value *BV = ShuffleBuilder.gather(GatheredScalars, BVMask.size()); @@ -14509,7 +14487,7 @@ void BoUpSLP::computeMinimumValueSizes() { return SIt != DemotedConsts.end() && is_contained(SIt->getSecond(), Idx); }) || - all_of(CTE->Scalars, Constant::classof)) + all_of(CTE->Scalars, IsaPred)) MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned); } } @@ -15257,12 +15235,10 @@ class HorizontalReduction { static Value *createOp(IRBuilderBase &Builder, RecurKind RdxKind, Value *LHS, Value *RHS, const Twine &Name, const ReductionOpsListType &ReductionOps) { - bool UseSelect = - ReductionOps.size() == 2 || - // Logical or/and. - (ReductionOps.size() == 1 && any_of(ReductionOps.front(), [](Value *V) { - return isa(V); - })); + bool UseSelect = ReductionOps.size() == 2 || + // Logical or/and. + (ReductionOps.size() == 1 && + any_of(ReductionOps.front(), IsaPred)); assert((!UseSelect || ReductionOps.size() != 2 || isa(ReductionOps[1][0])) && "Expected cmp + select pairs for reduction"); @@ -15501,7 +15477,7 @@ public: !hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst) || !isVectorizable(RdxKind, EdgeInst) || (R.isAnalyzedReductionRoot(EdgeInst) && - all_of(EdgeInst->operands(), Constant::classof))) { + all_of(EdgeInst->operands(), IsaPred))) { PossibleReducedVals.push_back(EdgeVal); continue; } @@ -16857,9 +16833,7 @@ bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI, SmallVector BuildVectorOpds; SmallVector Mask; if (!findBuildAggregate(IEI, TTI, BuildVectorOpds, BuildVectorInsts) || - (llvm::all_of( - BuildVectorOpds, - [](Value *V) { return isa(V); }) && + (llvm::all_of(BuildVectorOpds, IsaPred) && isFixedVectorShuffle(BuildVectorOpds, Mask))) return false; @@ -17080,10 +17054,7 @@ bool SLPVectorizerPass::vectorizeCmpInsts(iterator_range CmpInsts, bool SLPVectorizerPass::vectorizeInserts(InstSetVector &Instructions, BasicBlock *BB, BoUpSLP &R) { - assert(all_of(Instructions, - [](auto *I) { - return isa(I); - }) && + assert(all_of(Instructions, IsaPred) && "This function only accepts Insert instructions"); bool OpsChanged = false; SmallVector PostponedInsts; -- GitLab From 75f7d53f0ba5d77920ea895021b330f261e808cd Mon Sep 17 00:00:00 2001 From: Greg Clayton Date: Sun, 31 Mar 2024 12:08:59 -0700 Subject: [PATCH 0071/1690] Fix DWARF locations when we have large .dwp files. (#87164) We have the ability to load .dwp files with a .debug_info.dwo section that exceeds 4GB. There were 4 locations that were using 32 bit offsets and lengths to extract variable locations, and if a DIE was over the 4GB barrier, we would truncate the block offset for the variable locations and the variable expression would be garbage. This fixes the issues. It isn't possible to add a test for this as we don't want to create a 4GB .dwp file on test machines. --- .../Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index 1164bc62682a..49f13d2c89e3 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -3420,8 +3420,8 @@ static DWARFExpressionList GetExprListFromAtLocation(DWARFFormValue form_value, if (DWARFFormValue::IsBlockForm(form_value.Form())) { const DWARFDataExtractor &data = die.GetData(); - uint32_t block_offset = form_value.BlockData() - data.GetDataStart(); - uint32_t block_length = form_value.Unsigned(); + uint64_t block_offset = form_value.BlockData() - data.GetDataStart(); + uint64_t block_length = form_value.Unsigned(); return DWARFExpressionList( module, DataExtractor(data, block_offset, block_length), die.GetCU()); } @@ -3450,9 +3450,9 @@ GetExprListFromAtConstValue(DWARFFormValue form_value, ModuleSP module, const DWARFDataExtractor &debug_info_data = die.GetData(); if (DWARFFormValue::IsBlockForm(form_value.Form())) { // Retrieve the value as a block expression. - uint32_t block_offset = + uint64_t block_offset = form_value.BlockData() - debug_info_data.GetDataStart(); - uint32_t block_length = form_value.Unsigned(); + uint64_t block_length = form_value.Unsigned(); return DWARFExpressionList( module, DataExtractor(debug_info_data, block_offset, block_length), die.GetCU()); @@ -4061,8 +4061,8 @@ CollectCallSiteParameters(ModuleSP module, DWARFDIE call_site_die) { if (!DWARFFormValue::IsBlockForm(form_value.Form())) return {}; auto data = child.GetData(); - uint32_t block_offset = form_value.BlockData() - data.GetDataStart(); - uint32_t block_length = form_value.Unsigned(); + uint64_t block_offset = form_value.BlockData() - data.GetDataStart(); + uint64_t block_length = form_value.Unsigned(); return DWARFExpressionList( module, DataExtractor(data, block_offset, block_length), child.GetCU()); @@ -4167,8 +4167,8 @@ SymbolFileDWARF::CollectCallEdges(ModuleSP module, DWARFDIE function_die) { } auto data = child.GetData(); - uint32_t block_offset = form_value.BlockData() - data.GetDataStart(); - uint32_t block_length = form_value.Unsigned(); + uint64_t block_offset = form_value.BlockData() - data.GetDataStart(); + uint64_t block_length = form_value.Unsigned(); call_target = DWARFExpressionList( module, DataExtractor(data, block_offset, block_length), child.GetCU()); -- GitLab From 82c6eeed08b1c8267f6e92d594c910fe57a9775e Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Sun, 31 Mar 2024 21:28:05 +0200 Subject: [PATCH 0072/1690] [MLIR] Add a second map for registered OperationName in MLIRContext (NFC) (#87170) This speeds up registered op creation by 10-11% by allowing lookup by TypeID instead of StringRef. This can break your build/tests at runtime with an error that you're creating an unregistered operation that you have registered. If so you are likely using a class inheriting from the "real" operation. See for example in this patch the case of: class ConstantIndexOp : public arith::ConstantOp { If one is using `builder.create()` they actually create an `arith.constant` operation, but the builder will fetch the TypeID for the `ConstantIndexOp` class which does not correspond to any registered operation. To fix it the `ConstantIndexOp` class got this addition: static ::mlir::TypeID resolveTypeID() { return TypeID::get(); } --- mlir/include/mlir/Dialect/Arith/IR/Arith.h | 4 ++- .../Dialect/Transform/IR/TransformDialect.h | 8 ++--- mlir/include/mlir/IR/Builders.h | 2 +- mlir/include/mlir/IR/OpDefinition.h | 3 +- mlir/include/mlir/IR/OperationSupport.h | 5 ++++ mlir/lib/IR/MLIRContext.cpp | 29 ++++++++++++++----- 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/mlir/include/mlir/Dialect/Arith/IR/Arith.h b/mlir/include/mlir/Dialect/Arith/IR/Arith.h index 971c78f4a86a..00cdb13feb29 100644 --- a/mlir/include/mlir/Dialect/Arith/IR/Arith.h +++ b/mlir/include/mlir/Dialect/Arith/IR/Arith.h @@ -53,6 +53,7 @@ namespace arith { class ConstantIntOp : public arith::ConstantOp { public: using arith::ConstantOp::ConstantOp; + static ::mlir::TypeID resolveTypeID() { return TypeID::get(); } /// Build a constant int op that produces an integer of the specified width. static void build(OpBuilder &builder, OperationState &result, int64_t value, @@ -74,6 +75,7 @@ public: class ConstantFloatOp : public arith::ConstantOp { public: using arith::ConstantOp::ConstantOp; + static ::mlir::TypeID resolveTypeID() { return TypeID::get(); } /// Build a constant float op that produces a float of the specified type. static void build(OpBuilder &builder, OperationState &result, @@ -90,7 +92,7 @@ public: class ConstantIndexOp : public arith::ConstantOp { public: using arith::ConstantOp::ConstantOp; - + static ::mlir::TypeID resolveTypeID() { return TypeID::get(); } /// Build a constant int op that produces an index. static void build(OpBuilder &builder, OperationState &result, int64_t value); diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.h b/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.h index db27f2c6fc49..128eacdbe6ab 100644 --- a/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.h +++ b/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.h @@ -252,21 +252,21 @@ private: template void TransformDialect::addOperationIfNotRegistered() { - StringRef name = OpTy::getOperationName(); std::optional opName = - RegisteredOperationName::lookup(name, getContext()); + RegisteredOperationName::lookup(TypeID::get(), getContext()); if (!opName) { addOperations(); #ifndef NDEBUG + StringRef name = OpTy::getOperationName(); detail::checkImplementsTransformOpInterface(name, getContext()); #endif // NDEBUG return; } - if (opName->getTypeID() == TypeID::get()) + if (LLVM_LIKELY(opName->getTypeID() == TypeID::get())) return; - reportDuplicateOpRegistration(name); + reportDuplicateOpRegistration(OpTy::getOperationName()); } template diff --git a/mlir/include/mlir/IR/Builders.h b/mlir/include/mlir/IR/Builders.h index 43b6d2b38416..3beade017d1a 100644 --- a/mlir/include/mlir/IR/Builders.h +++ b/mlir/include/mlir/IR/Builders.h @@ -490,7 +490,7 @@ private: template RegisteredOperationName getCheckRegisteredInfo(MLIRContext *ctx) { std::optional opName = - RegisteredOperationName::lookup(OpT::getOperationName(), ctx); + RegisteredOperationName::lookup(TypeID::get(), ctx); if (LLVM_UNLIKELY(!opName)) { llvm::report_fatal_error( "Building op `" + OpT::getOperationName() + diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h index bd68c2744574..c177ae3594d1 100644 --- a/mlir/include/mlir/IR/OpDefinition.h +++ b/mlir/include/mlir/IR/OpDefinition.h @@ -1729,8 +1729,7 @@ public: template static void attachInterface(MLIRContext &context) { std::optional info = - RegisteredOperationName::lookup(ConcreteType::getOperationName(), - &context); + RegisteredOperationName::lookup(TypeID::get(), &context); if (!info) llvm::report_fatal_error( "Attempting to attach an interface to an unregistered operation " + diff --git a/mlir/include/mlir/IR/OperationSupport.h b/mlir/include/mlir/IR/OperationSupport.h index f2aa6cee8403..90e63ff8fcb3 100644 --- a/mlir/include/mlir/IR/OperationSupport.h +++ b/mlir/include/mlir/IR/OperationSupport.h @@ -676,6 +676,11 @@ public: static std::optional lookup(StringRef name, MLIRContext *ctx); + /// Lookup the registered operation information for the given operation. + /// Returns std::nullopt if the operation isn't registered. + static std::optional lookup(TypeID typeID, + MLIRContext *ctx); + /// Register a new operation in a Dialect object. /// This constructor is used by Dialect objects when they register the list /// of operations they contain. diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp index e1e6d14231d9..214b354c5347 100644 --- a/mlir/lib/IR/MLIRContext.cpp +++ b/mlir/lib/IR/MLIRContext.cpp @@ -183,7 +183,8 @@ public: llvm::StringMap> operations; /// A vector of operation info specifically for registered operations. - llvm::StringMap registeredOperations; + llvm::DenseMap registeredOperations; + llvm::StringMap registeredOperationsByName; /// This is a sorted container of registered operations for a deterministic /// and efficient `getRegisteredOperations` implementation. @@ -780,8 +781,8 @@ OperationName::OperationName(StringRef name, MLIRContext *context) { // Check the registered info map first. In the overwhelmingly common case, // the entry will be in here and it also removes the need to acquire any // locks. - auto registeredIt = ctxImpl.registeredOperations.find(name); - if (LLVM_LIKELY(registeredIt != ctxImpl.registeredOperations.end())) { + auto registeredIt = ctxImpl.registeredOperationsByName.find(name); + if (LLVM_LIKELY(registeredIt != ctxImpl.registeredOperationsByName.end())) { impl = registeredIt->second.impl; return; } @@ -909,10 +910,19 @@ OperationName::UnregisteredOpModel::hashProperties(OpaqueProperties prop) { //===----------------------------------------------------------------------===// std::optional -RegisteredOperationName::lookup(StringRef name, MLIRContext *ctx) { +RegisteredOperationName::lookup(TypeID typeID, MLIRContext *ctx) { auto &impl = ctx->getImpl(); - auto it = impl.registeredOperations.find(name); + auto it = impl.registeredOperations.find(typeID); if (it != impl.registeredOperations.end()) + return it->second; + return std::nullopt; +} + +std::optional +RegisteredOperationName::lookup(StringRef name, MLIRContext *ctx) { + auto &impl = ctx->getImpl(); + auto it = impl.registeredOperationsByName.find(name); + if (it != impl.registeredOperationsByName.end()) return it->getValue(); return std::nullopt; } @@ -945,11 +955,16 @@ void RegisteredOperationName::insert( // Update the registered info for this operation. auto emplaced = ctxImpl.registeredOperations.try_emplace( - name, RegisteredOperationName(impl)); + impl->getTypeID(), RegisteredOperationName(impl)); assert(emplaced.second && "operation name registration must be successful"); + auto emplacedByName = ctxImpl.registeredOperationsByName.try_emplace( + name, RegisteredOperationName(impl)); + (void)emplacedByName; + assert(emplacedByName.second && + "operation name registration must be successful"); // Add emplaced operation name to the sorted operations container. - RegisteredOperationName &value = emplaced.first->getValue(); + RegisteredOperationName &value = emplaced.first->second; ctxImpl.sortedRegisteredOperations.insert( llvm::upper_bound(ctxImpl.sortedRegisteredOperations, value, [](auto &lhs, auto &rhs) { -- GitLab From 23941019c0177b598c606581c8ca99eb7842769b Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Sun, 31 Mar 2024 23:25:51 +0200 Subject: [PATCH 0073/1690] Revert "[mlir]Fix dialect conversion drop uses" (#87205) Reverts llvm/llvm-project#86991 Some bots are broken with a leak being detected now. --- mlir/lib/Transforms/Utils/DialectConversion.cpp | 8 +++----- .../TosaToLinalg/tosa-to-linalg-invalid.mlir | 13 ------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index 3c72c8789e8e..2ec0b964b304 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -279,13 +279,11 @@ public: auto &blockOps = block->getOperations(); while (!blockOps.empty()) blockOps.remove(blockOps.begin()); - block->dropAllDefinedValueUses(); - if (block->getParent()) { + block->dropAllUses(); + if (block->getParent()) block->erase(); - } else { - block->dropAllDefinedValueUses(); + else delete block; - } } }; diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir index 6494e1b27194..17eec5936918 100644 --- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir +++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir @@ -15,16 +15,3 @@ func.func @tensor_with_unknown_rank(%arg0: tensor<*xi8>) -> tensor<*xi8> { %0 = "tosa.abs"(%arg0) : (tensor<*xi8>) -> tensor<*xi8> return %0 : tensor<*xi8> } - -// ----- - -// CHECK-LABEL: @unranked_add -func.func @unranked_add(%arg0 : tensor<10x10xf32> , %arg1 : tensor<10x10xf32>, %arg2 : tensor<*xf32>) -> (tensor<10x10xf32>) { - // expected-error@+3 {{failed to legalize operation 'tosa.add'}} - %reduce = tosa.reduce_max %arg0 {axis = 1 : i32} : (tensor<10x10xf32>) -> tensor<10x1xf32> - %1 = tosa.add %reduce, %arg1 : (tensor<10x1xf32>, tensor<10x10xf32>) -> tensor<10x10xf32> - %0 = tosa.add %1, %arg2 : (tensor<10x10xf32>, tensor<*xf32>) -> tensor<*xf32> - %2 = tosa.reshape %0 {new_shape = array} : (tensor<*xf32>) -> tensor<10x10xf32> - return %2 : tensor<10x10xf32> -} - -- GitLab From a0c019ae9ebd507bf7df58f6cc13eb7ebcd5e97f Mon Sep 17 00:00:00 2001 From: Philip Lassen Date: Sun, 31 Mar 2024 14:35:55 -0700 Subject: [PATCH 0074/1690] [mlir][linalg] Delete unused SameVariadicOperandSize trait from ops (#87124) Both `Transpose` and `Broadcast` specify the `SameVariadicOperandSize` trait. However neither has a variadic operand let alone more than one. This is likely a relic from copying the boilerplate of the `Reduce` definition. --- mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td | 2 -- 1 file changed, 2 deletions(-) diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td index 92d844eefb72..5ee363ed3257 100644 --- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td +++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td @@ -391,7 +391,6 @@ def ReduceOp : LinalgStructuredBase_Op<"reduce", [ def TransposeOp : LinalgStructuredBase_Op<"transpose", [ DeclareOpInterfaceMethods, - SameVariadicOperandSize, SingleBlockImplicitTerminator<"YieldOp">]> { let summary = "Transpose operator"; let description = [{ @@ -470,7 +469,6 @@ def TransposeOp : LinalgStructuredBase_Op<"transpose", [ def BroadcastOp : LinalgStructuredBase_Op<"broadcast", [ DeclareOpInterfaceMethods, - SameVariadicOperandSize, SingleBlockImplicitTerminator<"YieldOp">]> { let summary = "Static broadcast operator"; let description = [{ -- GitLab From ca68a14eb822d88bc32d511258f261e88dd6b7d6 Mon Sep 17 00:00:00 2001 From: Jon Chesterfield Date: Sun, 31 Mar 2024 23:17:05 +0100 Subject: [PATCH 0075/1690] Revert "Revert "Revert "[compiler-rt] Allow building builtins.a without a libc (#86737)""" This reverts commit d2b63ed0ca5c7eb3f8fc029e3a5705cf0ddab438. CI again. Leading theory is bad interaction between cmake and buildbot, some guesses at PR86737 --- compiler-rt/cmake/config-ix.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake index 911f48fa1381..46a6fdf8728f 100644 --- a/compiler-rt/cmake/config-ix.cmake +++ b/compiler-rt/cmake/config-ix.cmake @@ -235,9 +235,9 @@ set(COMPILER_RT_SUPPORTED_ARCH) # Try to compile a very simple source file to ensure we can target the given # platform. We use the results of these tests to build only the various target # runtime libraries supported by our current compilers cross-compiling -# abilities. Avoids using libc as that may not be available yet. +# abilities. set(SIMPLE_SOURCE ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/simple.cc) -file(WRITE ${SIMPLE_SOURCE} "int main(void) { return 0; }\n") +file(WRITE ${SIMPLE_SOURCE} "#include \n#include \nint main(void) { printf(\"hello, world\"); }\n") # Detect whether the current target platform is 32-bit or 64-bit, and setup # the correct commandline flags needed to attempt to target 32-bit and 64-bit. -- GitLab From f87bde2962dded71599034880f57f742ea371888 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Sun, 31 Mar 2024 17:25:28 -0700 Subject: [PATCH 0076/1690] [nfc] Update comment for raw profile version 10. (#87207) - The raw profile format change is in https://github.com/llvm/llvm-project/pull/82711 --- llvm/include/llvm/ProfileData/InstrProf.h | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index f6c7960ed8bc..3a71c02d6d5c 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -1195,6 +1195,7 @@ namespace RawInstrProf { // Version 7: Reorder binary id and include version in signature. // Version 8: Use relative counter pointer. // Version 9: Added relative bitmap bytes pointer and count used by MC/DC. +// Version 10: Added vtable, a new type of value profile data. const uint64_t Version = INSTR_PROF_RAW_VERSION; template inline uint64_t getMagic(); -- GitLab From 216b5e96664f72fdb63b6bbd6c422185c67ef818 Mon Sep 17 00:00:00 2001 From: "Ruiling, Song" Date: Mon, 1 Apr 2024 09:48:37 +0800 Subject: [PATCH 0077/1690] [AMDGPU] Expose RTZ version of f16 interpolation for gfx11+ (#86614) --- llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 18 +++++++++++ .../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 6 +++- llvm/lib/Target/AMDGPU/VINTERPInstructions.td | 6 ++++ .../GlobalISel/llvm.amdgcn.interp.inreg.ll | 30 +++++++++++++++++++ .../AMDGPU/llvm.amdgcn.interp.inreg.ll | 30 +++++++++++++++++++ 5 files changed, 89 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index bda3b066b776..3de20bb44e0c 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -2066,6 +2066,24 @@ def int_amdgcn_interp_inreg_p2_f16 : [IntrNoMem, IntrSpeculatable, ImmArg>]>; +// llvm.amdgcn.interp.p10.rtz.f16

, , , +// gfx11+ fp16 interpolation intrinsic, with round-toward-zero rounding mode. +// high selects whether high or low 16-bits are used for p and p0 operands +def int_amdgcn_interp_p10_rtz_f16: + DefaultAttrsIntrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty, llvm_i1_ty], + [IntrNoMem, IntrSpeculatable, + ImmArg>]>; + +// llvm.amdgcn.interp.p2.rtz.f16

, , , +// gfx11+ fp16 interpolation intrinsic, with round-toward-zero rounding mode. +// high selects whether high or low 16-bits are used for p operand +def int_amdgcn_interp_p2_rtz_f16 : + DefaultAttrsIntrinsic<[llvm_half_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty, llvm_i1_ty], + [IntrNoMem, IntrSpeculatable, + ImmArg>]>; + // Deprecated: use llvm.amdgcn.live.mask instead. def int_amdgcn_ps_live : DefaultAttrsIntrinsic < [llvm_i1_ty], diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp index a42e95f140ce..aa4ec785bf02 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp @@ -3135,6 +3135,8 @@ void AMDGPURegisterBankInfo::applyMappingImpl( case Intrinsic::amdgcn_interp_inreg_p2: case Intrinsic::amdgcn_interp_inreg_p10_f16: case Intrinsic::amdgcn_interp_inreg_p2_f16: + case Intrinsic::amdgcn_interp_p10_rtz_f16: + case Intrinsic::amdgcn_interp_p2_rtz_f16: applyDefaultMapping(OpdMapper); return; case Intrinsic::amdgcn_permlane16: @@ -4778,7 +4780,9 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case Intrinsic::amdgcn_interp_inreg_p10: case Intrinsic::amdgcn_interp_inreg_p2: case Intrinsic::amdgcn_interp_inreg_p10_f16: - case Intrinsic::amdgcn_interp_inreg_p2_f16: { + case Intrinsic::amdgcn_interp_inreg_p2_f16: + case Intrinsic::amdgcn_interp_p10_rtz_f16: + case Intrinsic::amdgcn_interp_p2_rtz_f16: { unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); diff --git a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td index 77063e2b70f6..1f7bffb26a60 100644 --- a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td +++ b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td @@ -173,6 +173,12 @@ defm : VInterpF16Pat; +defm : VInterpF16Pat; +defm : VInterpF16Pat; //===----------------------------------------------------------------------===// // VINTERP Real Instructions diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll index 623360f6b1d9..de46037e96e8 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll @@ -147,6 +147,34 @@ main_body: ret half %res } +define amdgpu_ps half @v_interp_rtz_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 { +; GCN-LABEL: v_interp_rtz_f16: +; GCN: ; %bb.0: ; %main_body +; GCN-NEXT: s_mov_b32 s3, exec_lo +; GCN-NEXT: s_wqm_b32 exec_lo, exec_lo +; GCN-NEXT: s_mov_b32 m0, s2 +; GCN-NEXT: lds_param_load v1, attr0.x wait_vdst:15 +; GCN-NEXT: s_mov_b32 exec_lo, s3 +; GCN-NEXT: v_mov_b32_e32 v0, s0 +; GCN-NEXT: v_mov_b32_e32 v2, s1 +; GCN-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GCN-NEXT: v_interp_p10_rtz_f16_f32 v3, v1, v0, v1 wait_exp:0 +; GCN-NEXT: v_interp_p10_rtz_f16_f32 v0, v1, v0, v1 op_sel:[1,0,1,0] wait_exp:7 +; GCN-NEXT: v_interp_p2_rtz_f16_f32 v3, v1, v2, v3 wait_exp:7 +; GCN-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GCN-NEXT: v_interp_p2_rtz_f16_f32 v0, v1, v2, v0 op_sel:[1,0,0,0] wait_exp:7 +; GCN-NEXT: v_add_f16_e32 v0, v3, v0 +; GCN-NEXT: ; return to shader part epilog +main_body: + %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0) + %l_p0 = call float @llvm.amdgcn.interp.p10.rtz.f16(float %p0, float %i, float %p0, i1 0) + %l_p1 = call half @llvm.amdgcn.interp.p2.rtz.f16(float %p0, float %j, float %l_p0, i1 0) + %h_p0 = call float @llvm.amdgcn.interp.p10.rtz.f16(float %p0, float %i, float %p0, i1 1) + %h_p1 = call half @llvm.amdgcn.interp.p2.rtz.f16(float %p0, float %j, float %h_p0, i1 1) + %res = fadd half %l_p1, %h_p1 + ret half %res +} + define amdgpu_ps half @v_interp_f16_imm_params(float inreg %i, float inreg %j) #0 { ; GCN-LABEL: v_interp_f16_imm_params: ; GCN: ; %bb.0: ; %main_body @@ -172,6 +200,8 @@ declare float @llvm.amdgcn.interp.inreg.p10(float, float, float) #0 declare float @llvm.amdgcn.interp.inreg.p2(float, float, float) #0 declare float @llvm.amdgcn.interp.inreg.p10.f16(float, float, float, i1) #0 declare half @llvm.amdgcn.interp.inreg.p2.f16(float, float, float, i1) #0 +declare float @llvm.amdgcn.interp.p10.rtz.f16(float, float, float, i1) #0 +declare half @llvm.amdgcn.interp.p2.rtz.f16(float, float, float, i1) #0 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0 declare void @llvm.amdgcn.exp.f16(i32, i32, float, float, float, float, i1, i1) #0 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll index 429528e9091d..e3dd036ecc30 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll @@ -147,6 +147,34 @@ main_body: ret half %res } +define amdgpu_ps half @v_interp_rtz_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 { +; GCN-LABEL: v_interp_rtz_f16: +; GCN: ; %bb.0: ; %main_body +; GCN-NEXT: s_mov_b32 s3, exec_lo +; GCN-NEXT: s_wqm_b32 exec_lo, exec_lo +; GCN-NEXT: s_mov_b32 m0, s2 +; GCN-NEXT: lds_param_load v1, attr0.x wait_vdst:15 +; GCN-NEXT: s_mov_b32 exec_lo, s3 +; GCN-NEXT: v_mov_b32_e32 v0, s0 +; GCN-NEXT: v_mov_b32_e32 v2, s1 +; GCN-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GCN-NEXT: v_interp_p10_rtz_f16_f32 v3, v1, v0, v1 wait_exp:0 +; GCN-NEXT: v_interp_p10_rtz_f16_f32 v0, v1, v0, v1 op_sel:[1,0,1,0] wait_exp:7 +; GCN-NEXT: v_interp_p2_rtz_f16_f32 v3, v1, v2, v3 wait_exp:7 +; GCN-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GCN-NEXT: v_interp_p2_rtz_f16_f32 v0, v1, v2, v0 op_sel:[1,0,0,0] wait_exp:7 +; GCN-NEXT: v_add_f16_e32 v0, v3, v0 +; GCN-NEXT: ; return to shader part epilog +main_body: + %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0) + %l_p0 = call float @llvm.amdgcn.interp.p10.rtz.f16(float %p0, float %i, float %p0, i1 0) + %l_p1 = call half @llvm.amdgcn.interp.p2.rtz.f16(float %p0, float %j, float %l_p0, i1 0) + %h_p0 = call float @llvm.amdgcn.interp.p10.rtz.f16(float %p0, float %i, float %p0, i1 1) + %h_p1 = call half @llvm.amdgcn.interp.p2.rtz.f16(float %p0, float %j, float %h_p0, i1 1) + %res = fadd half %l_p1, %h_p1 + ret half %res +} + define amdgpu_ps half @v_interp_f16_imm_params(float inreg %i, float inreg %j) #0 { ; GCN-LABEL: v_interp_f16_imm_params: ; GCN: ; %bb.0: ; %main_body @@ -172,6 +200,8 @@ declare float @llvm.amdgcn.interp.inreg.p10(float, float, float) #0 declare float @llvm.amdgcn.interp.inreg.p2(float, float, float) #0 declare float @llvm.amdgcn.interp.inreg.p10.f16(float, float, float, i1) #0 declare half @llvm.amdgcn.interp.inreg.p2.f16(float, float, float, i1) #0 +declare float @llvm.amdgcn.interp.p10.rtz.f16(float, float, float, i1) #0 +declare half @llvm.amdgcn.interp.p2.rtz.f16(float, float, float, i1) #0 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0 declare void @llvm.amdgcn.exp.f16(i32, i32, float, float, float, float, i1, i1) #0 -- GitLab From dc0cb17241aeff205a0c3a7cc4e790afab7484dd Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Sun, 31 Mar 2024 18:49:31 -0700 Subject: [PATCH 0078/1690] [msan] Nit: use init_origins parameter instead of __msan_get_track_origins() in InitShadowWithReExec (#86994) This fixes a nit I had accidentally introduced in https://github.com/llvm/llvm-project/pull/85142 I don't think the value of __msan_get_track_origins() will change between the start and end of InitShadowWithReExec, but it's cleaner to use the parameter. --- compiler-rt/lib/msan/msan_linux.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler-rt/lib/msan/msan_linux.cpp b/compiler-rt/lib/msan/msan_linux.cpp index cd2d9f5c720c..c68aec60ae13 100644 --- a/compiler-rt/lib/msan/msan_linux.cpp +++ b/compiler-rt/lib/msan/msan_linux.cpp @@ -175,7 +175,7 @@ bool InitShadowWithReExec(bool init_origins) { // Start with dry run: check layout is ok, but don't print warnings because // warning messages will cause tests to fail (even if we successfully re-exec // after the warning). - bool success = InitShadow(__msan_get_track_origins(), true); + bool success = InitShadow(init_origins, true); if (!success) { # if SANITIZER_LINUX // Perhaps ASLR entropy is too high. If ASLR is enabled, re-exec without it. @@ -197,7 +197,7 @@ bool InitShadowWithReExec(bool init_origins) { // The earlier dry run didn't actually map or protect anything. Run again in // non-dry run mode. - return success && InitShadow(__msan_get_track_origins(), false); + return success && InitShadow(init_origins, false); } static void MsanAtExit(void) { -- GitLab From fd38366e4525c5507bbb2a2fc1f7d113a964224e Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Sun, 31 Mar 2024 19:29:45 -0700 Subject: [PATCH 0079/1690] [BOLT][NFC] Clean includes, add license headers (#87200) --- bolt/include/bolt/Core/AddressMap.h | 1 - bolt/include/bolt/Core/BinaryData.h | 1 - bolt/include/bolt/Core/BinaryDomTree.h | 1 - bolt/include/bolt/Core/BinaryFunction.h | 1 - bolt/include/bolt/Core/BinaryLoop.h | 2 +- bolt/include/bolt/Core/BinarySection.h | 1 - bolt/include/bolt/Core/DebugData.h | 1 - bolt/include/bolt/Core/DebugNames.h | 2 +- bolt/include/bolt/Core/FunctionLayout.h | 1 - bolt/include/bolt/Core/MCPlus.h | 2 -- bolt/include/bolt/Core/MCPlusBuilder.h | 2 ++ bolt/include/bolt/Passes/BinaryPasses.h | 1 - bolt/include/bolt/Passes/CacheMetrics.h | 1 - bolt/include/bolt/Passes/DominatorAnalysis.h | 1 - bolt/include/bolt/Passes/ReachingDefOrUse.h | 2 -- bolt/include/bolt/Passes/ReachingInsns.h | 1 - bolt/include/bolt/Passes/ReorderUtils.h | 1 - bolt/include/bolt/Profile/ProfileReaderBase.h | 2 +- .../include/bolt/Profile/ProfileYAMLMapping.h | 1 - bolt/include/bolt/Rewrite/DWARFRewriter.h | 2 -- bolt/include/bolt/Rewrite/MetadataManager.h | 1 - bolt/include/bolt/Rewrite/RewriteInstance.h | 1 - .../include/bolt/RuntimeLibs/RuntimeLibrary.h | 1 - bolt/include/bolt/Utils/NameShortener.h | 1 - bolt/lib/Core/BinaryContext.cpp | 2 -- bolt/lib/Core/BinaryFunction.cpp | 2 ++ bolt/lib/Core/DIEBuilder.cpp | 1 - bolt/lib/Core/DebugData.cpp | 3 --- bolt/lib/Core/FunctionLayout.cpp | 13 +++++++---- bolt/lib/Core/HashUtilities.cpp | 1 - bolt/lib/Core/MCPlusBuilder.cpp | 1 - bolt/lib/Passes/CMOVConversion.cpp | 1 - bolt/lib/Passes/FixRISCVCallsPass.cpp | 8 +++++++ bolt/lib/Passes/FixRelaxationPass.cpp | 8 +++++++ bolt/lib/Passes/FrameOptimizer.cpp | 1 - bolt/lib/Passes/Hugify.cpp | 1 - bolt/lib/Passes/Inliner.cpp | 1 - bolt/lib/Passes/ShrinkWrapping.cpp | 1 - bolt/lib/Passes/SplitFunctions.cpp | 1 - bolt/lib/Passes/TailDuplication.cpp | 2 +- bolt/lib/Passes/ValidateInternalCalls.cpp | 1 - bolt/lib/Profile/CMakeLists.txt | 1 - bolt/lib/Profile/DataReader.cpp | 1 - bolt/lib/Profile/Heatmap.cpp | 1 - bolt/lib/Profile/ProfileReaderBase.cpp | 23 ------------------- bolt/lib/Rewrite/DWARFRewriter.cpp | 1 - bolt/lib/Rewrite/JITLinkLinker.cpp | 4 +++- bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp | 3 +-- bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp | 3 --- bolt/tools/bat-dump/bat-dump.cpp | 11 ++++++--- bolt/tools/heatmap/heatmap.cpp | 12 ++++++++-- bolt/unittests/Core/BinaryContext.cpp | 9 +++++++- bolt/unittests/Core/MCPlusBuilder.cpp | 9 +++++++- 53 files changed, 71 insertions(+), 86 deletions(-) delete mode 100644 bolt/lib/Profile/ProfileReaderBase.cpp diff --git a/bolt/include/bolt/Core/AddressMap.h b/bolt/include/bolt/Core/AddressMap.h index 85a9ab4473aa..31ed7d40ee7f 100644 --- a/bolt/include/bolt/Core/AddressMap.h +++ b/bolt/include/bolt/Core/AddressMap.h @@ -14,7 +14,6 @@ #ifndef BOLT_CORE_ADDRESS_MAP_H #define BOLT_CORE_ADDRESS_MAP_H -#include "llvm/ADT/StringRef.h" #include "llvm/MC/MCSymbol.h" #include diff --git a/bolt/include/bolt/Core/BinaryData.h b/bolt/include/bolt/Core/BinaryData.h index 5f1efda78190..495163f1b61a 100644 --- a/bolt/include/bolt/Core/BinaryData.h +++ b/bolt/include/bolt/Core/BinaryData.h @@ -18,7 +18,6 @@ #include "llvm/ADT/Twine.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Support/raw_ostream.h" -#include #include #include diff --git a/bolt/include/bolt/Core/BinaryDomTree.h b/bolt/include/bolt/Core/BinaryDomTree.h index a9565795f946..de27aa78769d 100644 --- a/bolt/include/bolt/Core/BinaryDomTree.h +++ b/bolt/include/bolt/Core/BinaryDomTree.h @@ -16,7 +16,6 @@ #include "bolt/Core/BinaryBasicBlock.h" #include "llvm/IR/Dominators.h" -#include "llvm/Support/GenericDomTreeConstruction.h" namespace llvm { namespace bolt { diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h index fd3eb2d87128..5089f8491280 100644 --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -52,7 +52,6 @@ #include #include #include -#include #include #include diff --git a/bolt/include/bolt/Core/BinaryLoop.h b/bolt/include/bolt/Core/BinaryLoop.h index 72dce77df8c1..b425c75715d8 100644 --- a/bolt/include/bolt/Core/BinaryLoop.h +++ b/bolt/include/bolt/Core/BinaryLoop.h @@ -15,7 +15,7 @@ #ifndef BOLT_CORE_BINARY_LOOP_H #define BOLT_CORE_BINARY_LOOP_H -#include "llvm/Support/GenericLoopInfoImpl.h" +#include "llvm/Support/GenericLoopInfo.h" namespace llvm { namespace bolt { diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h index 0f179877bd3d..5b7a5b08820e 100644 --- a/bolt/include/bolt/Core/BinarySection.h +++ b/bolt/include/bolt/Core/BinarySection.h @@ -18,7 +18,6 @@ #include "bolt/Core/DebugData.h" #include "bolt/Core/Relocation.h" #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/MachO.h" diff --git a/bolt/include/bolt/Core/DebugData.h b/bolt/include/bolt/Core/DebugData.h index 7d10b208dc83..166bb3617e57 100644 --- a/bolt/include/bolt/Core/DebugData.h +++ b/bolt/include/bolt/Core/DebugData.h @@ -27,7 +27,6 @@ #include #include #include -#include #include #include diff --git a/bolt/include/bolt/Core/DebugNames.h b/bolt/include/bolt/Core/DebugNames.h index fbaa7f4e68aa..a4fdde7c396a 100644 --- a/bolt/include/bolt/Core/DebugNames.h +++ b/bolt/include/bolt/Core/DebugNames.h @@ -14,7 +14,7 @@ #ifndef BOLT_CORE_DEBUG_NAMES_H #define BOLT_CORE_DEBUG_NAMES_H -#include "DebugData.h" +#include "bolt/Core/DebugData.h" #include "llvm/CodeGen/AccelTable.h" namespace llvm { diff --git a/bolt/include/bolt/Core/FunctionLayout.h b/bolt/include/bolt/Core/FunctionLayout.h index 2e4c184ba451..b685a99c79c1 100644 --- a/bolt/include/bolt/Core/FunctionLayout.h +++ b/bolt/include/bolt/Core/FunctionLayout.h @@ -25,7 +25,6 @@ #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include -#include namespace llvm { namespace bolt { diff --git a/bolt/include/bolt/Core/MCPlus.h b/bolt/include/bolt/Core/MCPlus.h index 1d2360c18033..601d70971286 100644 --- a/bolt/include/bolt/Core/MCPlus.h +++ b/bolt/include/bolt/Core/MCPlus.h @@ -14,10 +14,8 @@ #ifndef BOLT_CORE_MCPLUS_H #define BOLT_CORE_MCPLUS_H -#include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" -#include "llvm/Support/Casting.h" #include namespace llvm { diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h index 3818e3d66c99..f7614cf9ac97 100644 --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -19,6 +19,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/StringMap.h" +#include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCDisassembler/MCSymbolizer.h" #include "llvm/MC/MCExpr.h" @@ -27,6 +28,7 @@ #include "llvm/MC/MCInstrDesc.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/Support/Allocator.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/RWMutex.h" diff --git a/bolt/include/bolt/Passes/BinaryPasses.h b/bolt/include/bolt/Passes/BinaryPasses.h index 046765b16f19..8d89ef8b5484 100644 --- a/bolt/include/bolt/Passes/BinaryPasses.h +++ b/bolt/include/bolt/Passes/BinaryPasses.h @@ -18,7 +18,6 @@ #include "bolt/Core/DynoStats.h" #include "llvm/Support/CommandLine.h" #include -#include #include #include #include diff --git a/bolt/include/bolt/Passes/CacheMetrics.h b/bolt/include/bolt/Passes/CacheMetrics.h index 5c88d98c76c1..ea56d330446b 100644 --- a/bolt/include/bolt/Passes/CacheMetrics.h +++ b/bolt/include/bolt/Passes/CacheMetrics.h @@ -13,7 +13,6 @@ #ifndef BOLT_PASSES_CACHEMETRICS_H #define BOLT_PASSES_CACHEMETRICS_H -#include #include namespace llvm { diff --git a/bolt/include/bolt/Passes/DominatorAnalysis.h b/bolt/include/bolt/Passes/DominatorAnalysis.h index c2b5c3af0147..3f3afa943c06 100644 --- a/bolt/include/bolt/Passes/DominatorAnalysis.h +++ b/bolt/include/bolt/Passes/DominatorAnalysis.h @@ -11,7 +11,6 @@ #include "bolt/Passes/DataflowAnalysis.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Timer.h" namespace opts { extern llvm::cl::opt TimeOpts; diff --git a/bolt/include/bolt/Passes/ReachingDefOrUse.h b/bolt/include/bolt/Passes/ReachingDefOrUse.h index f38d1a373e18..585d673e3b84 100644 --- a/bolt/include/bolt/Passes/ReachingDefOrUse.h +++ b/bolt/include/bolt/Passes/ReachingDefOrUse.h @@ -11,9 +11,7 @@ #include "bolt/Passes/DataflowAnalysis.h" #include "bolt/Passes/RegAnalysis.h" -#include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Timer.h" #include namespace opts { diff --git a/bolt/include/bolt/Passes/ReachingInsns.h b/bolt/include/bolt/Passes/ReachingInsns.h index 65782b12064b..ef878f5e452d 100644 --- a/bolt/include/bolt/Passes/ReachingInsns.h +++ b/bolt/include/bolt/Passes/ReachingInsns.h @@ -11,7 +11,6 @@ #include "bolt/Passes/DataflowAnalysis.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Timer.h" namespace opts { extern llvm::cl::opt TimeOpts; diff --git a/bolt/include/bolt/Passes/ReorderUtils.h b/bolt/include/bolt/Passes/ReorderUtils.h index bc82b4f436fa..8ceb8ba62690 100644 --- a/bolt/include/bolt/Passes/ReorderUtils.h +++ b/bolt/include/bolt/Passes/ReorderUtils.h @@ -14,7 +14,6 @@ #ifndef BOLT_PASSES_REORDER_UTILS_H #define BOLT_PASSES_REORDER_UTILS_H -#include #include #include "llvm/ADT/BitVector.h" diff --git a/bolt/include/bolt/Profile/ProfileReaderBase.h b/bolt/include/bolt/Profile/ProfileReaderBase.h index 511718f3c0ec..3e5cf2612893 100644 --- a/bolt/include/bolt/Profile/ProfileReaderBase.h +++ b/bolt/include/bolt/Profile/ProfileReaderBase.h @@ -65,7 +65,7 @@ public: /// Return true if the function \p BF may have a profile available. /// The result is based on the name(s) of the function alone and the profile /// match is not guaranteed. - virtual bool mayHaveProfileData(const BinaryFunction &BF); + virtual bool mayHaveProfileData(const BinaryFunction &BF) { return true; } /// Return true if the profile contains an entry for a local object /// that has an associated file name. diff --git a/bolt/include/bolt/Profile/ProfileYAMLMapping.h b/bolt/include/bolt/Profile/ProfileYAMLMapping.h index 548b528ae2d6..9dd3920dbf09 100644 --- a/bolt/include/bolt/Profile/ProfileYAMLMapping.h +++ b/bolt/include/bolt/Profile/ProfileYAMLMapping.h @@ -14,7 +14,6 @@ #define BOLT_PROFILE_PROFILEYAMLMAPPING_H #include "bolt/Core/BinaryFunction.h" -#include "llvm/ADT/StringRef.h" #include "llvm/Support/YAMLTraits.h" #include diff --git a/bolt/include/bolt/Rewrite/DWARFRewriter.h b/bolt/include/bolt/Rewrite/DWARFRewriter.h index 20972f3d0b85..2c482bd2b9ea 100644 --- a/bolt/include/bolt/Rewrite/DWARFRewriter.h +++ b/bolt/include/bolt/Rewrite/DWARFRewriter.h @@ -22,9 +22,7 @@ #include #include #include -#include #include -#include #include namespace llvm { diff --git a/bolt/include/bolt/Rewrite/MetadataManager.h b/bolt/include/bolt/Rewrite/MetadataManager.h index efbc74b4daba..2ff70dbaab3d 100644 --- a/bolt/include/bolt/Rewrite/MetadataManager.h +++ b/bolt/include/bolt/Rewrite/MetadataManager.h @@ -11,7 +11,6 @@ #include "bolt/Rewrite/MetadataRewriter.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/Support/Error.h" namespace llvm { namespace bolt { diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h index 97ab65cd5a4a..826677cd63b2 100644 --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -17,7 +17,6 @@ #include "bolt/Core/Linker.h" #include "bolt/Rewrite/MetadataManager.h" #include "bolt/Utils/NameResolver.h" -#include "llvm/ADT/ArrayRef.h" #include "llvm/MC/StringTableBuilder.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/ObjectFile.h" diff --git a/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h b/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h index c845cb7f7b21..e392029156bc 100644 --- a/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h +++ b/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h @@ -17,7 +17,6 @@ #include "bolt/Core/Linker.h" #include "llvm/ADT/StringRef.h" -#include #include namespace llvm { diff --git a/bolt/include/bolt/Utils/NameShortener.h b/bolt/include/bolt/Utils/NameShortener.h index 9c7b7ec9ba65..fd61235f93c8 100644 --- a/bolt/include/bolt/Utils/NameShortener.h +++ b/bolt/include/bolt/Utils/NameShortener.h @@ -14,7 +14,6 @@ #define BOLT_UTILS_NAME_SHORTENER_H #include "llvm/ADT/StringMap.h" -#include "llvm/ADT/Twine.h" namespace llvm { namespace bolt { diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp index 7b474bc40dac..47eae964e816 100644 --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -14,7 +14,6 @@ #include "bolt/Core/BinaryEmitter.h" #include "bolt/Core/BinaryFunction.h" #include "bolt/Utils/CommandLineOpts.h" -#include "bolt/Utils/NameResolver.h" #include "bolt/Utils/Utils.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Twine.h" @@ -39,7 +38,6 @@ #include #include #include -#include #include using namespace llvm; diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index 24f21cb39329..1fa96dfaabde 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -34,6 +34,8 @@ #include "llvm/Object/ObjectFile.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/GenericDomTreeConstruction.h" +#include "llvm/Support/GenericLoopInfoImpl.h" #include "llvm/Support/GraphWriter.h" #include "llvm/Support/LEB128.h" #include "llvm/Support/Regex.h" diff --git a/bolt/lib/Core/DIEBuilder.cpp b/bolt/lib/Core/DIEBuilder.cpp index 354fe5059443..c4b0b251c120 100644 --- a/bolt/lib/Core/DIEBuilder.cpp +++ b/bolt/lib/Core/DIEBuilder.cpp @@ -22,7 +22,6 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Format.h" #include "llvm/Support/LEB128.h" #include diff --git a/bolt/lib/Core/DebugData.cpp b/bolt/lib/Core/DebugData.cpp index a75016ede309..a987a103a08b 100644 --- a/bolt/lib/Core/DebugData.cpp +++ b/bolt/lib/Core/DebugData.cpp @@ -13,7 +13,6 @@ #include "bolt/Core/DebugData.h" #include "bolt/Core/BinaryContext.h" #include "bolt/Core/DIEBuilder.h" -#include "bolt/Rewrite/RewriteInstance.h" #include "bolt/Utils/Utils.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/CodeGen/DIE.h" @@ -23,7 +22,6 @@ #include "llvm/MC/MCAssembler.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCObjectStreamer.h" -#include "llvm/Support/Allocator.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/EndianStream.h" #include "llvm/Support/LEB128.h" @@ -32,7 +30,6 @@ #include #include #include -#include #include #include #include diff --git a/bolt/lib/Core/FunctionLayout.cpp b/bolt/lib/Core/FunctionLayout.cpp index 27e40de94be6..73f4d5247d9a 100644 --- a/bolt/lib/Core/FunctionLayout.cpp +++ b/bolt/lib/Core/FunctionLayout.cpp @@ -1,12 +1,17 @@ +//===- bolt/Core/FunctionLayout.cpp - Fragmented Function Layout -*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + #include "bolt/Core/FunctionLayout.h" -#include "bolt/Core/BinaryFunction.h" +#include "bolt/Core/BinaryBasicBlock.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/edit_distance.h" #include -#include -#include #include -#include using namespace llvm; using namespace bolt; diff --git a/bolt/lib/Core/HashUtilities.cpp b/bolt/lib/Core/HashUtilities.cpp index d40159b2e216..c4c67bd68198 100644 --- a/bolt/lib/Core/HashUtilities.cpp +++ b/bolt/lib/Core/HashUtilities.cpp @@ -12,7 +12,6 @@ #include "bolt/Core/HashUtilities.h" #include "bolt/Core/BinaryContext.h" -#include "bolt/Core/BinaryFunction.h" #include "llvm/MC/MCInstPrinter.h" namespace llvm { diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp index f22b35a90bc5..7ff7a2288451 100644 --- a/bolt/lib/Core/MCPlusBuilder.cpp +++ b/bolt/lib/Core/MCPlusBuilder.cpp @@ -22,7 +22,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include -#include #define DEBUG_TYPE "mcplus" diff --git a/bolt/lib/Passes/CMOVConversion.cpp b/bolt/lib/Passes/CMOVConversion.cpp index 2492ff217946..cdd99b55207e 100644 --- a/bolt/lib/Passes/CMOVConversion.cpp +++ b/bolt/lib/Passes/CMOVConversion.cpp @@ -17,7 +17,6 @@ #include "llvm/ADT/PostOrderIterator.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" -#include #define DEBUG_TYPE "cmov" diff --git a/bolt/lib/Passes/FixRISCVCallsPass.cpp b/bolt/lib/Passes/FixRISCVCallsPass.cpp index 83c745facb29..9011ef303a80 100644 --- a/bolt/lib/Passes/FixRISCVCallsPass.cpp +++ b/bolt/lib/Passes/FixRISCVCallsPass.cpp @@ -1,3 +1,11 @@ +//===- bolt/Passes/FixRISCVCallsPass.cpp ------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + #include "bolt/Passes/FixRISCVCallsPass.h" #include "bolt/Core/ParallelUtilities.h" diff --git a/bolt/lib/Passes/FixRelaxationPass.cpp b/bolt/lib/Passes/FixRelaxationPass.cpp index a49fb9894e80..7c970e464a94 100644 --- a/bolt/lib/Passes/FixRelaxationPass.cpp +++ b/bolt/lib/Passes/FixRelaxationPass.cpp @@ -1,3 +1,11 @@ +//===- bolt/Passes/FixRelaxationPass.cpp ------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + #include "bolt/Passes/FixRelaxationPass.h" #include "bolt/Core/ParallelUtilities.h" diff --git a/bolt/lib/Passes/FrameOptimizer.cpp b/bolt/lib/Passes/FrameOptimizer.cpp index fb5f8eafa5cf..8461225e1819 100644 --- a/bolt/lib/Passes/FrameOptimizer.cpp +++ b/bolt/lib/Passes/FrameOptimizer.cpp @@ -20,7 +20,6 @@ #include "bolt/Utils/CommandLineOpts.h" #include "llvm/Support/Timer.h" #include -#include #define DEBUG_TYPE "fop" diff --git a/bolt/lib/Passes/Hugify.cpp b/bolt/lib/Passes/Hugify.cpp index b77356153bfd..1ac1b08573b8 100644 --- a/bolt/lib/Passes/Hugify.cpp +++ b/bolt/lib/Passes/Hugify.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "bolt/Passes/Hugify.h" -#include "llvm/Support/CommandLine.h" #define DEBUG_TYPE "bolt-hugify" diff --git a/bolt/lib/Passes/Inliner.cpp b/bolt/lib/Passes/Inliner.cpp index a3b2017aa32a..84e7d97067b0 100644 --- a/bolt/lib/Passes/Inliner.cpp +++ b/bolt/lib/Passes/Inliner.cpp @@ -27,7 +27,6 @@ #include "bolt/Passes/Inliner.h" #include "bolt/Core/MCPlus.h" #include "llvm/Support/CommandLine.h" -#include #define DEBUG_TYPE "bolt-inliner" diff --git a/bolt/lib/Passes/ShrinkWrapping.cpp b/bolt/lib/Passes/ShrinkWrapping.cpp index c9706500758d..176321c58dc9 100644 --- a/bolt/lib/Passes/ShrinkWrapping.cpp +++ b/bolt/lib/Passes/ShrinkWrapping.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// #include "bolt/Passes/ShrinkWrapping.h" -#include "bolt/Core/MCPlus.h" #include "bolt/Passes/DataflowInfoManager.h" #include "bolt/Passes/MCF.h" #include "bolt/Utils/CommandLineOpts.h" diff --git a/bolt/lib/Passes/SplitFunctions.cpp b/bolt/lib/Passes/SplitFunctions.cpp index cdbb2a15f667..f9e634d15a97 100644 --- a/bolt/lib/Passes/SplitFunctions.cpp +++ b/bolt/lib/Passes/SplitFunctions.cpp @@ -17,7 +17,6 @@ #include "bolt/Core/ParallelUtilities.h" #include "bolt/Utils/CommandLineOpts.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/Sequence.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Support/CommandLine.h" diff --git a/bolt/lib/Passes/TailDuplication.cpp b/bolt/lib/Passes/TailDuplication.cpp index 2163e3a6a008..463ea49527fa 100644 --- a/bolt/lib/Passes/TailDuplication.cpp +++ b/bolt/lib/Passes/TailDuplication.cpp @@ -13,9 +13,9 @@ #include "bolt/Passes/TailDuplication.h" #include "llvm/ADT/DenseMap.h" #include "llvm/MC/MCRegisterInfo.h" -#include #include +#include #define DEBUG_TYPE "taildup" diff --git a/bolt/lib/Passes/ValidateInternalCalls.cpp b/bolt/lib/Passes/ValidateInternalCalls.cpp index 54ae621159cf..88df2e5b59f3 100644 --- a/bolt/lib/Passes/ValidateInternalCalls.cpp +++ b/bolt/lib/Passes/ValidateInternalCalls.cpp @@ -14,7 +14,6 @@ #include "bolt/Core/BinaryBasicBlock.h" #include "bolt/Passes/DataflowInfoManager.h" #include "bolt/Passes/FrameAnalysis.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/MC/MCInstPrinter.h" #include #include diff --git a/bolt/lib/Profile/CMakeLists.txt b/bolt/lib/Profile/CMakeLists.txt index 3a31a9cc1919..045ac47edb95 100644 --- a/bolt/lib/Profile/CMakeLists.txt +++ b/bolt/lib/Profile/CMakeLists.txt @@ -3,7 +3,6 @@ add_llvm_library(LLVMBOLTProfile DataAggregator.cpp DataReader.cpp Heatmap.cpp - ProfileReaderBase.cpp StaleProfileMatching.cpp YAMLProfileReader.cpp YAMLProfileWriter.cpp diff --git a/bolt/lib/Profile/DataReader.cpp b/bolt/lib/Profile/DataReader.cpp index aa21eb121ad6..67f357fe4d3f 100644 --- a/bolt/lib/Profile/DataReader.cpp +++ b/bolt/lib/Profile/DataReader.cpp @@ -18,7 +18,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Errc.h" -#include #undef DEBUG_TYPE #define DEBUG_TYPE "bolt-prof" diff --git a/bolt/lib/Profile/Heatmap.cpp b/bolt/lib/Profile/Heatmap.cpp index 13541f6f6a4b..210a5cc98c10 100644 --- a/bolt/lib/Profile/Heatmap.cpp +++ b/bolt/lib/Profile/Heatmap.cpp @@ -10,7 +10,6 @@ #include "bolt/Utils/CommandLineOpts.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/Twine.h" -#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Format.h" diff --git a/bolt/lib/Profile/ProfileReaderBase.cpp b/bolt/lib/Profile/ProfileReaderBase.cpp deleted file mode 100644 index ee6166b1da36..000000000000 --- a/bolt/lib/Profile/ProfileReaderBase.cpp +++ /dev/null @@ -1,23 +0,0 @@ -//===- bolt/Profile/ProfileReaderBase.cpp ---------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Interface to be implemented by all profile readers. -// -//===----------------------------------------------------------------------===// - -#include "bolt/Profile/ProfileReaderBase.h" - -namespace llvm { -namespace bolt { - -bool ProfileReaderBase::mayHaveProfileData(const BinaryFunction &BF) { - return true; -} - -} // namespace bolt -} // namespace llvm diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp index 6a6f5708cdf7..feeba89a40dc 100644 --- a/bolt/lib/Rewrite/DWARFRewriter.cpp +++ b/bolt/lib/Rewrite/DWARFRewriter.cpp @@ -14,7 +14,6 @@ #include "bolt/Core/DynoStats.h" #include "bolt/Core/ParallelUtilities.h" #include "bolt/Rewrite/RewriteInstance.h" -#include "bolt/Utils/Utils.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" diff --git a/bolt/lib/Rewrite/JITLinkLinker.cpp b/bolt/lib/Rewrite/JITLinkLinker.cpp index 66e129bf1d05..be8f9dd03467 100644 --- a/bolt/lib/Rewrite/JITLinkLinker.cpp +++ b/bolt/lib/Rewrite/JITLinkLinker.cpp @@ -5,9 +5,11 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// + #include "bolt/Rewrite/JITLinkLinker.h" +#include "bolt/Core/BinaryContext.h" #include "bolt/Core/BinaryData.h" -#include "bolt/Rewrite/RewriteInstance.h" +#include "bolt/Core/BinarySection.h" #include "llvm/ExecutionEngine/JITLink/ELF_riscv.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" diff --git a/bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp b/bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp index b6fc49d3f5d6..d114d70f2d37 100644 --- a/bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp +++ b/bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp @@ -11,10 +11,9 @@ //===----------------------------------------------------------------------===// #include "bolt/RuntimeLibs/HugifyRuntimeLibrary.h" -#include "bolt/Core/BinaryFunction.h" +#include "bolt/Core/BinaryContext.h" #include "bolt/Core/Linker.h" #include "llvm/MC/MCStreamer.h" -#include "llvm/Support/Alignment.h" #include "llvm/Support/CommandLine.h" using namespace llvm; diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp index ab9623d5c51b..74f2f0aae91e 100644 --- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp +++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp @@ -16,10 +16,7 @@ #include "llvm/BinaryFormat/ELF.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Format.h" -#include "llvm/Support/raw_ostream.h" #define DEBUG_TYPE "mcplus" diff --git a/bolt/tools/bat-dump/bat-dump.cpp b/bolt/tools/bat-dump/bat-dump.cpp index 2e9b26cc137a..709eb076bca2 100644 --- a/bolt/tools/bat-dump/bat-dump.cpp +++ b/bolt/tools/bat-dump/bat-dump.cpp @@ -1,9 +1,16 @@ +//===- bolt/tools/bat-dump/bat-dump.cpp - BAT dumper utility --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + #include "bolt/Profile/BoltAddressTranslation.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" -#include "llvm/ADT/iterator_range.h" #include "llvm/Object/Binary.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/Error.h" @@ -18,7 +25,6 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/Program.h" #include "llvm/Support/raw_ostream.h" -#include #include #include #include @@ -27,7 +33,6 @@ #include #include #include -#include using namespace llvm; using namespace bolt; diff --git a/bolt/tools/heatmap/heatmap.cpp b/bolt/tools/heatmap/heatmap.cpp index 9b190dd288b2..3bb9f2ce7491 100644 --- a/bolt/tools/heatmap/heatmap.cpp +++ b/bolt/tools/heatmap/heatmap.cpp @@ -1,4 +1,11 @@ -#include "bolt/Profile/DataAggregator.h" +//===- bolt/tools/heatmap/heatmap.cpp - Profile heatmap visualization tool ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + #include "bolt/Rewrite/RewriteInstance.h" #include "bolt/Utils/CommandLineOpts.h" #include "llvm/MC/TargetRegistry.h" @@ -6,7 +13,8 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" -#include "llvm/Support/Path.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Program.h" #include "llvm/Support/TargetSelect.h" using namespace llvm; diff --git a/bolt/unittests/Core/BinaryContext.cpp b/bolt/unittests/Core/BinaryContext.cpp index 6737ba27bfcd..cfec72a34a59 100644 --- a/bolt/unittests/Core/BinaryContext.cpp +++ b/bolt/unittests/Core/BinaryContext.cpp @@ -1,7 +1,14 @@ +//===- bolt/unittest/Core/BinaryContext.cpp -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + #include "bolt/Core/BinaryContext.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" -#include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/TargetSelect.h" #include "gtest/gtest.h" diff --git a/bolt/unittests/Core/MCPlusBuilder.cpp b/bolt/unittests/Core/MCPlusBuilder.cpp index 2fbfc792e42e..62f3aaab4a72 100644 --- a/bolt/unittests/Core/MCPlusBuilder.cpp +++ b/bolt/unittests/Core/MCPlusBuilder.cpp @@ -1,3 +1,11 @@ +//===- bolt/unittest/Core/MCPlusBuilder.cpp -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + #ifdef AARCH64_AVAILABLE #include "AArch64Subtarget.h" #endif // AARCH64_AVAILABLE @@ -11,7 +19,6 @@ #include "bolt/Rewrite/RewriteInstance.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" -#include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/TargetSelect.h" #include "gtest/gtest.h" -- GitLab From 2bfb19e813fcfdcb4a37b16bd77d267d7c77f0f8 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 31 Mar 2024 19:45:12 -0700 Subject: [PATCH 0080/1690] Revert "Make two texts static in `ReplayInlineAdvisor`" (#82071) Reverts llvm/llvm-project#79489 We know that the issues was with asan/annotations. We can revert it. --- llvm/lib/Analysis/ReplayInlineAdvisor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp index 0814483db343..2ca02eb17417 100644 --- a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp +++ b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp @@ -43,8 +43,8 @@ ReplayInlineAdvisor::ReplayInlineAdvisor( // main:3:1.1; // We use the callsite string after `at callsite` to replay inlining. line_iterator LineIt(*BufferOrErr.get(), /*SkipBlanks=*/true); - static const std::string PositiveRemark = "' inlined into '"; - static const std::string NegativeRemark = "' will not be inlined into '"; + const std::string PositiveRemark = "' inlined into '"; + const std::string NegativeRemark = "' will not be inlined into '"; for (; !LineIt.is_at_eof(); ++LineIt) { StringRef Line = *LineIt; -- GitLab From 70deb7bfe90af91c68454b70683fbe98feaea87d Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Mon, 1 Apr 2024 10:57:54 +0800 Subject: [PATCH 0081/1690] [BPF] expand cttz, ctlz for i32, i64 (#73668) Fixes: https://github.com/llvm/llvm-project/issues/62252 Depends on: #73667 --- llvm/lib/Target/BPF/BPFISelLowering.cpp | 9 +- llvm/test/CodeGen/BPF/cttz-ctlz.ll | 304 ++++++++++++++++++++++++ 2 files changed, 308 insertions(+), 5 deletions(-) create mode 100644 llvm/test/CodeGen/BPF/cttz-ctlz.ll diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp index b8ca8ec98c06..071fe004806e 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.cpp +++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp @@ -113,6 +113,10 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, setOperationAction(ISD::SRL_PARTS, VT, Expand); setOperationAction(ISD::SRA_PARTS, VT, Expand); setOperationAction(ISD::CTPOP, VT, Expand); + setOperationAction(ISD::CTTZ, VT, Expand); + setOperationAction(ISD::CTLZ, VT, Expand); + setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); + setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); setOperationAction(ISD::SETCC, VT, Expand); setOperationAction(ISD::SELECT, VT, Expand); @@ -125,11 +129,6 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, STI.getHasJmp32() ? Custom : Promote); } - setOperationAction(ISD::CTTZ, MVT::i64, Custom); - setOperationAction(ISD::CTLZ, MVT::i64, Custom); - setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Custom); - setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom); - setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); if (!STI.hasMovsx()) { setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand); diff --git a/llvm/test/CodeGen/BPF/cttz-ctlz.ll b/llvm/test/CodeGen/BPF/cttz-ctlz.ll new file mode 100644 index 000000000000..f42b2e2d1087 --- /dev/null +++ b/llvm/test/CodeGen/BPF/cttz-ctlz.ll @@ -0,0 +1,304 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -march=bpf | FileCheck %s + +; test that we can expand CTTZ & CTLZ + +declare i32 @llvm.cttz.i32(i32, i1) + +define i32 @cttz_i32_zdef(i32 %a) { +; CHECK-LABEL: cttz_i32_zdef: +; CHECK: # %bb.0: +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 = -r2 +; CHECK-NEXT: r1 &= r2 +; CHECK-NEXT: r1 *= 125613361 +; CHECK-NEXT: r2 = 4160749568 ll +; CHECK-NEXT: r1 &= r2 +; CHECK-NEXT: r1 >>= 27 +; CHECK-NEXT: r2 = {{\.?LCPI[0-9]+_[0-9]+}} ll +; CHECK-NEXT: r2 += r1 +; CHECK-NEXT: r0 = *(u8 *)(r2 + 0) +; CHECK-NEXT: exit + %ret = call i32 @llvm.cttz.i32(i32 %a, i1 1) + ret i32 %ret +} + + +define i32 @cttz_i32(i32 %a) { +; CHECK-LABEL: cttz_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: r0 = 32 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 <<= 32 +; CHECK-NEXT: r2 >>= 32 +; CHECK-NEXT: if r2 == 0 goto LBB1_2 +; CHECK-NEXT: # %bb.1: # %cond.false +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 = -r2 +; CHECK-NEXT: r1 &= r2 +; CHECK-NEXT: r1 *= 125613361 +; CHECK-NEXT: r2 = 4160749568 ll +; CHECK-NEXT: r1 &= r2 +; CHECK-NEXT: r1 >>= 27 +; CHECK-NEXT: r2 = {{\.?LCPI[0-9]+_[0-9]+}} ll +; CHECK-NEXT: r2 += r1 +; CHECK-NEXT: r0 = *(u8 *)(r2 + 0) +; CHECK-NEXT: LBB1_2: # %cond.end +; CHECK-NEXT: exit + %ret = call i32 @llvm.cttz.i32(i32 %a, i1 0) + ret i32 %ret +} + +declare i64 @llvm.cttz.i64(i64, i1) + +define i64 @cttz_i64_zdef(i64 %a) { +; CHECK-LABEL: cttz_i64_zdef: +; CHECK: # %bb.0: +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 = -r2 +; CHECK-NEXT: r1 &= r2 +; CHECK-NEXT: r2 = 151050438420815295 ll +; CHECK-NEXT: r1 *= r2 +; CHECK-NEXT: r1 >>= 58 +; CHECK-NEXT: r2 = {{\.?LCPI[0-9]+_[0-9]+}} ll +; CHECK-NEXT: r2 += r1 +; CHECK-NEXT: r0 = *(u8 *)(r2 + 0) +; CHECK-NEXT: exit + %ret = call i64 @llvm.cttz.i64(i64 %a, i1 1) + ret i64 %ret +} + + +define i64 @cttz_i64(i64 %a) { +; CHECK-LABEL: cttz_i64: +; CHECK: # %bb.0: +; CHECK-NEXT: r0 = 64 +; CHECK-NEXT: if r1 == 0 goto LBB3_2 +; CHECK-NEXT: # %bb.1: # %cond.false +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 = -r2 +; CHECK-NEXT: r1 &= r2 +; CHECK-NEXT: r2 = 151050438420815295 ll +; CHECK-NEXT: r1 *= r2 +; CHECK-NEXT: r1 >>= 58 +; CHECK-NEXT: r2 = {{\.?LCPI[0-9]+_[0-9]+}} ll +; CHECK-NEXT: r2 += r1 +; CHECK-NEXT: r0 = *(u8 *)(r2 + 0) +; CHECK-NEXT: LBB3_2: # %cond.end +; CHECK-NEXT: exit + %ret = call i64 @llvm.cttz.i64(i64 %a, i1 0) + ret i64 %ret +} + + +declare i32 @llvm.ctlz.i32(i32, i1) + +define i32 @ctlz_i32_zdef(i32 %a) { +; CHECK-LABEL: ctlz_i32_zdef: +; CHECK: # %bb.0: +; CHECK-NEXT: r2 = 4294967294 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r3 >>= 1 +; CHECK-NEXT: r1 |= r3 +; CHECK-NEXT: r2 = 4294967292 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r3 >>= 2 +; CHECK-NEXT: r1 |= r3 +; CHECK-NEXT: r2 = 4294967280 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r3 >>= 4 +; CHECK-NEXT: r1 |= r3 +; CHECK-NEXT: r2 = 4294967040 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r3 >>= 8 +; CHECK-NEXT: r1 |= r3 +; CHECK-NEXT: r2 = 4294901760 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r3 >>= 16 +; CHECK-NEXT: r1 |= r3 +; CHECK-NEXT: r1 ^= -1 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 1 +; CHECK-NEXT: r2 &= 1431655765 +; CHECK-NEXT: r1 -= r2 +; CHECK-NEXT: r0 = r1 +; CHECK-NEXT: r0 &= 858993459 +; CHECK-NEXT: r1 >>= 2 +; CHECK-NEXT: r1 &= 858993459 +; CHECK-NEXT: r0 += r1 +; CHECK-NEXT: r1 = r0 +; CHECK-NEXT: r1 >>= 4 +; CHECK-NEXT: r0 += r1 +; CHECK-NEXT: r0 &= 252645135 +; CHECK-NEXT: r0 *= 16843009 +; CHECK-NEXT: r1 = 4278190080 ll +; CHECK-NEXT: r0 &= r1 +; CHECK-NEXT: r0 >>= 24 +; CHECK-NEXT: exit + %ret = call i32 @llvm.ctlz.i32(i32 %a, i1 1) + ret i32 %ret +} + + +define i32 @ctlz_i32(i32 %a) { +; CHECK-LABEL: ctlz_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: r0 = 32 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 <<= 32 +; CHECK-NEXT: r2 >>= 32 +; CHECK-NEXT: if r2 == 0 goto LBB5_2 +; CHECK-NEXT: # %bb.1: # %cond.false +; CHECK-NEXT: r2 = 4294967294 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r3 >>= 1 +; CHECK-NEXT: r1 |= r3 +; CHECK-NEXT: r2 = 4294967292 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r3 >>= 2 +; CHECK-NEXT: r1 |= r3 +; CHECK-NEXT: r2 = 4294967280 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r3 >>= 4 +; CHECK-NEXT: r1 |= r3 +; CHECK-NEXT: r2 = 4294967040 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r3 >>= 8 +; CHECK-NEXT: r1 |= r3 +; CHECK-NEXT: r2 = 4294901760 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r3 >>= 16 +; CHECK-NEXT: r1 |= r3 +; CHECK-NEXT: r1 ^= -1 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 1 +; CHECK-NEXT: r2 &= 1431655765 +; CHECK-NEXT: r1 -= r2 +; CHECK-NEXT: r0 = r1 +; CHECK-NEXT: r0 &= 858993459 +; CHECK-NEXT: r1 >>= 2 +; CHECK-NEXT: r1 &= 858993459 +; CHECK-NEXT: r0 += r1 +; CHECK-NEXT: r1 = r0 +; CHECK-NEXT: r1 >>= 4 +; CHECK-NEXT: r0 += r1 +; CHECK-NEXT: r0 &= 252645135 +; CHECK-NEXT: r0 *= 16843009 +; CHECK-NEXT: r1 = 4278190080 ll +; CHECK-NEXT: r0 &= r1 +; CHECK-NEXT: r0 >>= 24 +; CHECK-NEXT: LBB5_2: # %cond.end +; CHECK-NEXT: exit + %ret = call i32 @llvm.ctlz.i32(i32 %a, i1 0) + ret i32 %ret +} + +declare i64 @llvm.ctlz.i64(i64, i1) + +define i64 @ctlz_i64_zdef(i64 %a) { +; CHECK-LABEL: ctlz_i64_zdef: +; CHECK: # %bb.0: +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 1 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 2 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 4 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 8 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 16 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 32 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r1 ^= -1 +; CHECK-NEXT: r2 = 6148914691236517205 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 >>= 1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r1 -= r3 +; CHECK-NEXT: r2 = 3689348814741910323 ll +; CHECK-NEXT: r0 = r1 +; CHECK-NEXT: r0 &= r2 +; CHECK-NEXT: r1 >>= 2 +; CHECK-NEXT: r1 &= r2 +; CHECK-NEXT: r0 += r1 +; CHECK-NEXT: r1 = r0 +; CHECK-NEXT: r1 >>= 4 +; CHECK-NEXT: r0 += r1 +; CHECK-NEXT: r1 = 1085102592571150095 ll +; CHECK-NEXT: r0 &= r1 +; CHECK-NEXT: r1 = 72340172838076673 ll +; CHECK-NEXT: r0 *= r1 +; CHECK-NEXT: r0 >>= 56 +; CHECK-NEXT: exit + %ret = call i64 @llvm.ctlz.i64(i64 %a, i1 1) + ret i64 %ret +} + + +define i64 @ctlz_i64(i64 %a) { +; CHECK-LABEL: ctlz_i64: +; CHECK: # %bb.0: +; CHECK-NEXT: r0 = 64 +; CHECK-NEXT: if r1 == 0 goto LBB7_2 +; CHECK-NEXT: # %bb.1: # %cond.false +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 1 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 2 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 4 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 8 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 16 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r2 = r1 +; CHECK-NEXT: r2 >>= 32 +; CHECK-NEXT: r1 |= r2 +; CHECK-NEXT: r1 ^= -1 +; CHECK-NEXT: r2 = 6148914691236517205 ll +; CHECK-NEXT: r3 = r1 +; CHECK-NEXT: r3 >>= 1 +; CHECK-NEXT: r3 &= r2 +; CHECK-NEXT: r1 -= r3 +; CHECK-NEXT: r2 = 3689348814741910323 ll +; CHECK-NEXT: r0 = r1 +; CHECK-NEXT: r0 &= r2 +; CHECK-NEXT: r1 >>= 2 +; CHECK-NEXT: r1 &= r2 +; CHECK-NEXT: r0 += r1 +; CHECK-NEXT: r1 = r0 +; CHECK-NEXT: r1 >>= 4 +; CHECK-NEXT: r0 += r1 +; CHECK-NEXT: r1 = 1085102592571150095 ll +; CHECK-NEXT: r0 &= r1 +; CHECK-NEXT: r1 = 72340172838076673 ll +; CHECK-NEXT: r0 *= r1 +; CHECK-NEXT: r0 >>= 56 +; CHECK-NEXT: LBB7_2: # %cond.end +; CHECK-NEXT: exit + %ret = call i64 @llvm.ctlz.i64(i64 %a, i1 0) + ret i64 %ret +} + -- GitLab From 90c738ef15fc6f10255dced0b1557f2cd2c4f43b Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 31 Mar 2024 21:23:55 -0700 Subject: [PATCH 0082/1690] [IR] Introduce `llvm.allow.{runtime,ubsan}.check()` (#84850) The goal is to have ability to change logic compile time based on PGO data. Our primary application is removing UBSAN checks from hot code. Then we'd like to use this for libc++ hardening and regular debug asserts. Previous attempt is #84214. Benefits from special intrinsic vs #84214: 1. Resulting binary is 3% faster than removing traps (on "test-suite/MultiSource/Benchmarks" with PGO+ThinLTO) 2. Intrinsic can be used from source code to change behavior from C/C++ program. E.g. enabling asserts in cold code. 3. Easier to match basic blocks. RFC: https://discourse.llvm.org/t/rfc-add-llvm-experimental-hot-intrinsic-or-llvm-hot/77641 --------- Co-authored-by: Nikita Popov --- llvm/docs/LangRef.rst | 114 +++++++++++++++++++++++++++++ llvm/include/llvm/IR/Intrinsics.td | 10 +++ 2 files changed, 124 insertions(+) diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 845254aa6e50..1d4ff5238226 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -27839,6 +27839,120 @@ constant `true`. However it is always correct to replace it with any other `i1` value. Any pass can freely do it if it can benefit from non-default lowering. +'``llvm.allow.ubsan.check``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare i1 @llvm.allow.ubsan.check(i8 immarg %kind) + +Overview: +""""""""" + +This intrinsic returns ``true`` if and only if the compiler opted to enable the +ubsan check in the current basic block. + +Rules to allow ubsan checks are not part of the intrinsic declaration, and +controlled by compiler options. + +This intrinsic is the ubsan specific version of ``@llvm.allow.runtime.check()``. + +Arguments: +"""""""""" + +An integer describing the kind of ubsan check guarded by the intrinsic. + +Semantics: +"""""""""" + +The intrinsic ``@llvm.allow.ubsan.check()`` returns either ``true`` or +``false``, depending on compiler options. + +For each evaluation of a call to this intrinsic, the program must be valid and +correct both if it returns ``true`` and if it returns ``false``. + +When used in a branch condition, it selects one of the two paths: + +* `true``: Executes the UBSan check and reports any failures. + +* `false`: Bypasses the check, assuming it always succeeds. + +Example: + +.. code-block:: text + + %allow = call i1 @llvm.allow.ubsan.check(i8 5) + %not.allow = xor i1 %allow, true + %cond = or i1 %ubcheck, %not.allow + br i1 %cond, label %cont, label %trap + + cont: + ; Proceed + + trap: + call void @llvm.ubsantrap(i8 5) + unreachable + + +'``llvm.allow.runtime.check``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare i1 @llvm.allow.runtime.check(metadata %kind) + +Overview: +""""""""" + +This intrinsic returns ``true`` if and only if the compiler opted to enable +runtime checks in the current basic block. + +Rules to allow runtime checks are not part of the intrinsic declaration, and +controlled by compiler options. + +This intrinsic is non-ubsan specific version of ``@llvm.allow.ubsan.check()``. + +Arguments: +"""""""""" + +A string identifying the kind of runtime check guarded by the intrinsic. The +string can be used to control rules to allow checks. + +Semantics: +"""""""""" + +The intrinsic ``@llvm.allow.runtime.check()`` returns either ``true`` or +``false``, depending on compiler options. + +For each evaluation of a call to this intrinsic, the program must be valid and +correct both if it returns ``true`` and if it returns ``false``. + +When used in a branch condition, it allows us to choose between +two alternative correct solutions for the same problem. + +If the intrinsic is evaluated as ``true``, program should execute a guarded +check. If the intrinsic is evaluated as ``false``, the program should avoid any +unnecessary checks. + +Example: + +.. code-block:: text + + %allow = call i1 @llvm.allow.runtime.check(metadata !"my_check") + br i1 %allow, label %fast_path, label %slow_path + + fast_path: + ; Omit diagnostics. + + slow_path: + ; Additional diagnostics. + '``llvm.load.relative``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 764902426f0b..c04f4c526921 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1731,6 +1731,16 @@ def int_debugtrap : Intrinsic<[]>, def int_ubsantrap : Intrinsic<[], [llvm_i8_ty], [IntrNoReturn, IntrCold, ImmArg>]>; +// Return true if ubsan check is allowed. +def int_allow_ubsan_check : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i8_ty], + [IntrInaccessibleMemOnly, IntrWriteMem, ImmArg>, NoUndef]>, + ClangBuiltin<"__builtin_allow_ubsan_check">; + +// Return true if runtime check is allowed. +def int_allow_runtime_check : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_metadata_ty], + [IntrInaccessibleMemOnly, IntrWriteMem, NoUndef]>, + ClangBuiltin<"__builtin_allow_runtime_check">; + // Support for dynamic deoptimization (or de-specialization) def int_experimental_deoptimize : Intrinsic<[llvm_any_ty], [llvm_vararg_ty], [Throws]>; -- GitLab From a21e0ba192147af4968742785f375dba0e3ea768 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 31 Mar 2024 21:24:05 -0700 Subject: [PATCH 0083/1690] [clang-tidy] Fix buffer overflow in compareHeaders (#87213) `RHS` can be shorter than `LHS`. Reported by asan after #83440. --- clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp b/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp index b6d9c50d0b10..a44720c47eca 100644 --- a/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp +++ b/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp @@ -108,7 +108,7 @@ int compareHeaders(StringRef LHS, StringRef RHS, IncludeSorter::IncludeStyle Style) { if (Style == IncludeSorter::IncludeStyle::IS_Google_ObjC) { const std::pair &Mismatch = - std::mismatch(LHS.begin(), LHS.end(), RHS.begin()); + std::mismatch(LHS.begin(), LHS.end(), RHS.begin(), RHS.end()); if ((Mismatch.first != LHS.end()) && (Mismatch.second != RHS.end())) { if ((*Mismatch.first == '.') && (*Mismatch.second == '+')) { return -1; -- GitLab From 20f56e1f8e51d672425ec0c8f2ec243b131e8296 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 31 Mar 2024 22:19:33 -0700 Subject: [PATCH 0084/1690] [CodeGen] Add default lowering for llvm.allow.{runtime,ubsan}.check() (#86049) RFC: https://discourse.llvm.org/t/rfc-add-llvm-experimental-hot-intrinsic-or-llvm-hot/77641 --- llvm/lib/CodeGen/CodeGenPrepare.cpp | 4 ++- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 5 +++ llvm/lib/CodeGen/IntrinsicLowering.cpp | 5 +++ llvm/lib/CodeGen/SelectionDAG/FastISel.cpp | 4 +++ .../SelectionDAG/SelectionDAGBuilder.cpp | 5 +++ llvm/test/CodeGen/AArch64/allow-check.ll | 30 +++++++++++++++++ llvm/test/CodeGen/AMDGPU/allow-check.ll | 30 +++++++++++++++++ llvm/test/CodeGen/Generic/allow-check.ll | 23 +++++++++++++ llvm/test/CodeGen/RISCV/allow-check.ll | 32 +++++++++++++++++++ llvm/test/CodeGen/X86/allow-check.ll | 28 ++++++++++++++++ 10 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/AArch64/allow-check.ll create mode 100644 llvm/test/CodeGen/AMDGPU/allow-check.ll create mode 100644 llvm/test/CodeGen/Generic/allow-check.ll create mode 100644 llvm/test/CodeGen/RISCV/allow-check.ll create mode 100644 llvm/test/CodeGen/X86/allow-check.ll diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index 9f99bb7e693f..e657872c3828 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -2462,8 +2462,10 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT) { break; case Intrinsic::assume: llvm_unreachable("llvm.assume should have been removed already"); + case Intrinsic::allow_runtime_check: + case Intrinsic::allow_ubsan_check: case Intrinsic::experimental_widenable_condition: { - // Give up on future widening oppurtunties so that we can fold away dead + // Give up on future widening opportunities so that we can fold away dead // paths and merge blocks before going into block-local instruction // selection. if (II->use_empty()) { diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 80b3f5825922..47e980e05281 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2495,6 +2495,11 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, return translateTrap(CI, MIRBuilder, TargetOpcode::G_DEBUGTRAP); case Intrinsic::ubsantrap: return translateTrap(CI, MIRBuilder, TargetOpcode::G_UBSANTRAP); + case Intrinsic::allow_runtime_check: + case Intrinsic::allow_ubsan_check: + MIRBuilder.buildCopy(getOrCreateVReg(CI), + getOrCreateVReg(*ConstantInt::getTrue(CI.getType()))); + return true; case Intrinsic::amdgcn_cs_chain: return translateCallBase(CI, MIRBuilder); case Intrinsic::fptrunc_round: { diff --git a/llvm/lib/CodeGen/IntrinsicLowering.cpp b/llvm/lib/CodeGen/IntrinsicLowering.cpp index 09d282d12c5f..45fba4341ad0 100644 --- a/llvm/lib/CodeGen/IntrinsicLowering.cpp +++ b/llvm/lib/CodeGen/IntrinsicLowering.cpp @@ -243,6 +243,11 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) { break; } + case Intrinsic::allow_runtime_check: + case Intrinsic::allow_ubsan_check: + CI->replaceAllUsesWith(ConstantInt::getTrue(CI->getType())); + return; + case Intrinsic::ctpop: CI->replaceAllUsesWith(LowerCTPOP(Context, CI->getArgOperand(0), CI)); break; diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp index 51562400b77d..dae827883b29 100644 --- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1461,6 +1461,10 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) { case Intrinsic::is_constant: llvm_unreachable("llvm.is.constant.* should have been lowered already"); + case Intrinsic::allow_runtime_check: + case Intrinsic::allow_ubsan_check: + llvm_unreachable("llvm.*.check should have been lowered already"); + case Intrinsic::launder_invariant_group: case Intrinsic::strip_invariant_group: case Intrinsic::expect: { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 5db32472d197..3c8456efd08b 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -7326,6 +7326,11 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, return; } + case Intrinsic::allow_runtime_check: + case Intrinsic::allow_ubsan_check: + setValue(&I, getValue(ConstantInt::getTrue(I.getType()))); + return; + case Intrinsic::uadd_with_overflow: case Intrinsic::sadd_with_overflow: case Intrinsic::usub_with_overflow: diff --git a/llvm/test/CodeGen/AArch64/allow-check.ll b/llvm/test/CodeGen/AArch64/allow-check.ll new file mode 100644 index 000000000000..9e4a47357906 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/allow-check.ll @@ -0,0 +1,30 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=aarch64 -global-isel=0 -fast-isel=0 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -global-isel=1 -fast-isel=0 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -global-isel=0 -fast-isel=1 | FileCheck %s + +target triple = "aarch64-linux" + +define i1 @test_runtime() local_unnamed_addr { +; CHECK-LABEL: test_runtime: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w0, #1 // =0x1 +; CHECK-NEXT: ret +entry: + %allow = call i1 @llvm.allow.runtime.check(metadata !"test_check") + ret i1 %allow +} + +declare i1 @llvm.allow.runtime.check(metadata) nounwind + +define i1 @test_ubsan() local_unnamed_addr { +; CHECK-LABEL: test_ubsan: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w0, #1 // =0x1 +; CHECK-NEXT: ret +entry: + %allow = call i1 @llvm.allow.ubsan.check(i8 7) + ret i1 %allow +} + +declare i1 @llvm.allow.ubsan.check(i8) nounwind diff --git a/llvm/test/CodeGen/AMDGPU/allow-check.ll b/llvm/test/CodeGen/AMDGPU/allow-check.ll new file mode 100644 index 000000000000..d4f5621ce26a --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/allow-check.ll @@ -0,0 +1,30 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -global-isel=0 -fast-isel=0 | FileCheck %s +; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -global-isel=1 -fast-isel=0 | FileCheck %s +; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -global-isel=0 -fast-isel=1 | FileCheck %s + +define i1 @test_runtime() local_unnamed_addr { +; CHECK-LABEL: test_runtime: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_mov_b32_e32 v0, 1 +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %allow = call i1 @llvm.allow.runtime.check(metadata !"test_check") + ret i1 %allow +} + +declare i1 @llvm.allow.runtime.check(metadata) nounwind + +define i1 @test_ubsan() local_unnamed_addr { +; CHECK-LABEL: test_ubsan: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_mov_b32_e32 v0, 1 +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %allow = call i1 @llvm.allow.ubsan.check(i8 7) + ret i1 %allow +} + +declare i1 @llvm.allow.ubsan.check(i8) nounwind diff --git a/llvm/test/CodeGen/Generic/allow-check.ll b/llvm/test/CodeGen/Generic/allow-check.ll new file mode 100644 index 000000000000..7c6cb16eacca --- /dev/null +++ b/llvm/test/CodeGen/Generic/allow-check.ll @@ -0,0 +1,23 @@ +; RUN: llc < %s -O3 -global-isel=0 -fast-isel=0 +; RUN: llc < %s -O3 -global-isel=1 -fast-isel=0 +; RUN: llc < %s -O3 -global-isel=0 -fast-isel=1 + +; RUN: llc < %s -O0 -global-isel=0 -fast-isel=0 +; RUN: llc < %s -O0 -global-isel=1 -fast-isel=0 +; RUN: llc < %s -O0 -global-isel=0 -fast-isel=1 + +define i1 @test_runtime() local_unnamed_addr { +entry: + %allow = call i1 @llvm.allow.runtime.check(metadata !"test_check") + ret i1 %allow +} + +declare i1 @llvm.allow.runtime.check(metadata) nounwind + +define i1 @test_ubsan() local_unnamed_addr { +entry: + %allow = call i1 @llvm.allow.ubsan.check(i8 7) + ret i1 %allow +} + +declare i1 @llvm.allow.ubsan.check(i8) nounwind diff --git a/llvm/test/CodeGen/RISCV/allow-check.ll b/llvm/test/CodeGen/RISCV/allow-check.ll new file mode 100644 index 000000000000..0ddb5266db8f --- /dev/null +++ b/llvm/test/CodeGen/RISCV/allow-check.ll @@ -0,0 +1,32 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=riscv32 -global-isel=0 -fast-isel=0 | FileCheck %s +; RUN: llc < %s -mtriple=riscv32 -global-isel=1 -fast-isel=0 | FileCheck %s +; RUN: llc < %s -mtriple=riscv32 -global-isel=0 -fast-isel=1 | FileCheck %s + +; RUN: llc < %s -mtriple=riscv64 -global-isel=0 -fast-isel=0 | FileCheck %s +; RUN: llc < %s -mtriple=riscv64 -global-isel=1 -fast-isel=0 | FileCheck %s +; RUN: llc < %s -mtriple=riscv64 -global-isel=0 -fast-isel=1 | FileCheck %s + +define i1 @test_runtime() local_unnamed_addr { +; CHECK-LABEL: test_runtime: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a0, 1 +; CHECK-NEXT: ret +entry: + %allow = call i1 @llvm.allow.runtime.check(metadata !"test_check") + ret i1 %allow +} + +declare i1 @llvm.allow.runtime.check(metadata) nounwind + +define i1 @test_ubsan() local_unnamed_addr { +; CHECK-LABEL: test_ubsan: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a0, 1 +; CHECK-NEXT: ret +entry: + %allow = call i1 @llvm.allow.ubsan.check(i8 7) + ret i1 %allow +} + +declare i1 @llvm.allow.ubsan.check(i8) nounwind diff --git a/llvm/test/CodeGen/X86/allow-check.ll b/llvm/test/CodeGen/X86/allow-check.ll new file mode 100644 index 000000000000..602e5a94ee3e --- /dev/null +++ b/llvm/test/CodeGen/X86/allow-check.ll @@ -0,0 +1,28 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=x86_64 -global-isel=0 -fast-isel=0 | FileCheck %s +; RUN: llc < %s -mtriple=x86_64 -global-isel=1 -fast-isel=0 | FileCheck %s +; RUN: llc < %s -mtriple=x86_64 -global-isel=0 -fast-isel=1 | FileCheck %s + +define i1 @test_runtime() local_unnamed_addr { +; CHECK-LABEL: test_runtime: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movb $1, %al +; CHECK-NEXT: retq +entry: + %allow = call i1 @llvm.allow.runtime.check(metadata !"test_check") + ret i1 %allow +} + +declare i1 @llvm.allow.runtime.check(metadata) nounwind + +define i1 @test_ubsan() local_unnamed_addr { +; CHECK-LABEL: test_ubsan: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movb $1, %al +; CHECK-NEXT: retq +entry: + %allow = call i1 @llvm.allow.ubsan.check(i8 7) + ret i1 %allow +} + +declare i1 @llvm.allow.ubsan.check(i8) nounwind -- GitLab From 421557974a3e0f469e6f4c3caecbf8aba69bb5bf Mon Sep 17 00:00:00 2001 From: Sameer Sahasrabuddhe Date: Mon, 1 Apr 2024 10:51:13 +0530 Subject: [PATCH 0085/1690] [AMDGPU] Use glue for convergence tokens at call-like operations (#86766) The earlier implementation on AMDGPU used explicit token operands at SI_CALL and SI_CALL_ISEL. This is now replaced with CONVERGENCECTRL_GLUE operands, with the following effects: - The treatment of tokens at call-like operations is now consistent with the treatment at intrinsics. - Support for tail calls using implicit tokens at SI_TCRETURN "just works". - The extra parameter at call-like instructions is eliminated, thus restoring those instructions and their handling to the original state. The new glue node is placed after the existing glue node for the outgoing call parameters, which seems to not interfere with selection of the call-like nodes. --- .../SelectionDAG/SelectionDAGBuilder.cpp | 2 - llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 34 ++++---- llvm/lib/Target/AMDGPU/SIInstructions.td | 8 +- .../test/CodeGen/AMDGPU/convergence-tokens.ll | 12 +-- .../CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll | 18 ----- .../kernel-vgpr-spill-mubuf-with-voffset.ll | 1 - .../AMDGPU/need-fp-from-vgpr-spills.ll | 18 ++--- .../AMDGPU/no-source-locations-in-prologue.ll | 1 - .../AMDGPU/sgpr-spills-split-regalloc.ll | 15 ++-- .../CodeGen/AMDGPU/stacksave_stackrestore.ll | 78 +++++++++---------- llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 26 +++---- .../CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll | 1 - .../AMDGPU/whole-wave-register-spill.ll | 1 - .../test/CodeGen/AMDGPU/wwm-reserved-spill.ll | 2 - llvm/test/CodeGen/AMDGPU/wwm-reserved.ll | 4 - 15 files changed, 81 insertions(+), 140 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 3c8456efd08b..618bdee7f405 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -8612,8 +8612,6 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) { auto *Token = Bundle->Inputs[0].get(); ConvControlToken = getValue(Token); - } else { - ConvControlToken = DAG.getUNDEF(MVT::Untyped); } TargetLowering::CallLoweringInfo CLI(DAG); diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 0a4370de0613..14948ef9ea8d 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -3833,9 +3833,6 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64)); } - if (!IsTailCall) - Ops.push_back(CLI.ConvergenceControlToken); - if (IsTailCall) { // Each tail call may have to adjust the stack by a different amount, so // this information must travel along with the operation for eventual @@ -3862,6 +3859,17 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, if (InGlue.getNode()) Ops.push_back(InGlue); + // NOTE: This potentially results in *two* glue operands, and the wrong one + // might possibly show up where the other was intended. In particular, + // Emitter::EmitMachineNode() expects only the glued convergence token if it + // exists. Similarly, the selection of the call expects to match only the + // InGlue operand if it exists. + if (SDValue Token = CLI.ConvergenceControlToken) { + Ops.push_back(SDValue(DAG.getMachineNode(TargetOpcode::CONVERGENCECTRL_GLUE, + DL, MVT::Glue, Token), + 0)); + } + SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); // If we're doing a tall call, use a TC_RETURN here rather than an @@ -5226,24 +5234,8 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( MachineInstrBuilder MIB; MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL), ReturnAddrReg); - for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { - MachineOperand &MO = MI.getOperand(I); - if (I != 2) { - MIB.add(MO); - continue; - } - } - - MachineOperand &MO = MI.getOperand(2); - MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); - // The token operand is always a register, whose definition is IMPLICIT_DEF - // iff there was no token on the call. - if (MachineInstr *Def = MRI.getVRegDef(MO.getReg())) { - if (Def->getOpcode() != TargetOpcode::IMPLICIT_DEF) { - MO.setImplicit(); - MIB.add(MO); - } - } + for (const MachineOperand &MO : MI.operands()) + MIB.add(MO); MIB.cloneMemRefs(MI); MI.eraseFromParent(); diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index 1c942dcefdac..04f3a2f57605 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -618,8 +618,8 @@ def SI_RETURN : SPseudoInstSI < // This version is only needed so we can fill in the output register // in the custom inserter. def SI_CALL_ISEL : SPseudoInstSI < - (outs), (ins SSrc_b64:$src0, unknown:$callee, unknown:$token), - [(AMDGPUcall i64:$src0, tglobaladdr:$callee, untyped:$token)]> { + (outs), (ins SSrc_b64:$src0, unknown:$callee), + [(AMDGPUcall i64:$src0, tglobaladdr:$callee)]> { let Size = 4; let isCall = 1; let SchedRW = [WriteBranch]; @@ -629,8 +629,8 @@ def SI_CALL_ISEL : SPseudoInstSI < } def : GCNPat< - (AMDGPUcall i64:$src0, (i64 0), untyped:$token), - (SI_CALL_ISEL $src0, (i64 0), untyped:$token) + (AMDGPUcall i64:$src0, (i64 0)), + (SI_CALL_ISEL $src0, (i64 0)) >; // Wrapper around s_swappc_b64 with extra $callee parameter to track diff --git a/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll b/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll index 6beccce9400e..1c8725f52f7e 100644 --- a/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll +++ b/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll @@ -4,7 +4,7 @@ ; CHECK-LABEL: name: basic_call ; CHECK: [[TOKEN:%[0-9]+]]{{[^ ]*}} = CONVERGENCECTRL_ENTRY -; ISEL: {{.*}} SI_CALL_ISEL {{.*}}, @foo, [[TOKEN]], csr_amdgpu, {{.*}} +; ISEL: {{.*}} SI_CALL_ISEL {{.*}}, @foo, csr_amdgpu, {{.*}}, implicit [[TOKEN]] ; DEADMI: {{.*}} SI_CALL {{.*}}, @foo, csr_amdgpu, {{.*}}, implicit [[TOKEN]] ; GISEL: {{.*}} G_SI_CALL {{.*}}, @foo, csr_amdgpu, {{.*}}, implicit [[TOKEN]] define i32 @basic_call(i32 %src) #0 { @@ -92,15 +92,9 @@ define i32 @nested(i32 %src) #0 { ret i32 %sum } -; COM: FIXME: Tokens on tail-call have not been implemented for SelectionDAG -; COM: yet; the corresponding checks have been commented out. -; ; CHECK-LABEL: name: tail_call_void_func_void -; GISEL: [[TOKEN:%[0-9]+]]{{[^ ]*}} = CONVERGENCECTRL_ENTRY -; COM: CHECK: [[TOKEN:%[0-9]+]]{{[^ ]*}} = CONVERGENCECTRL_ENTRY -; COM: ISEL: {{.*}} SI_CALL_ISEL {{.*}}, @external_void_func_void, [[TOKEN]], csr_amdgpu, {{.*}} -; COM: DEADMI: {{.*}} SI_CALL {{.*}}, @external_void_func_void, csr_amdgpu, {{.*}}, implicit [[TOKEN]] -; GISEL: {{.*}} SI_TCRETURN {{.*}}, @external_void_func_void, 0, csr_amdgpu, implicit [[TOKEN]] +; CHECK: [[TOKEN:%[0-9]+]]{{[^ ]*}} = CONVERGENCECTRL_ENTRY +; CHECK: {{.*}} SI_TCRETURN {{.*}}, @external_void_func_void, 0, csr_amdgpu, {{.*}}implicit [[TOKEN]] define void @tail_call_void_func_void() #0 { %t1 = call token @llvm.experimental.convergence.entry() tail call void @external_void_func_void() [ "convergencectrl"(token %t1) ] diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll index e015095a4884..ab160ffc10ed 100644 --- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll +++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll @@ -92,7 +92,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc(<4 x i32> inreg %a, <4 x i32> %b ; DAGISEL-GFX11-NEXT: $vgpr5 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr6 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr7 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -122,7 +121,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc(<4 x i32> inreg %a, <4 x i32> %b ; DAGISEL-GFX10-NEXT: $vgpr5 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr6 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr7 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -234,7 +232,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_ptr(ptr inreg %a, ptr %b, ptr ad ; DAGISEL-GFX11-NEXT: $vgpr9 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr10 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr11 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -272,7 +269,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_ptr(ptr inreg %a, ptr %b, ptr ad ; DAGISEL-GFX10-NEXT: $vgpr9 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr10 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr11 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -404,7 +400,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_struct( {ptr, i32, <4 x i32>} in ; DAGISEL-GFX11-NEXT: $vgpr11 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr12 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr13 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF2:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -454,7 +449,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_struct( {ptr, i32, <4 x i32>} in ; DAGISEL-GFX10-NEXT: $vgpr11 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr12 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr13 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF2:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -506,7 +500,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_float(float inreg %a, float %b) ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -524,7 +517,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_float(float inreg %a, float %b) ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -576,7 +568,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) { ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -594,7 +585,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) { ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -646,7 +636,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat % ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -664,7 +653,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat % ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -716,7 +704,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_i16(i16 inreg %a, i16 %b) { ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -734,7 +721,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_i16(i16 inreg %a, i16 %b) { ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -870,7 +856,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_v16i16(<16 x i16> inreg %a, <16 ; DAGISEL-GFX11-NEXT: $vgpr13 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr14 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr15 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -916,7 +901,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_v16i16(<16 x i16> inreg %a, <16 ; DAGISEL-GFX10-NEXT: $vgpr13 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr14 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr15 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -2480,7 +2464,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_many_regs(<36 x i32> inreg %a, <128 ; DAGISEL-GFX11-NEXT: $vgpr29 = COPY [[COPY134]] ; DAGISEL-GFX11-NEXT: $vgpr30 = COPY [[COPY133]] ; DAGISEL-GFX11-NEXT: $vgpr31 = COPY [[COPY132]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $vgpr31 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 528, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -2827,7 +2810,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_many_regs(<36 x i32> inreg %a, <128 ; DAGISEL-GFX10-NEXT: $vgpr29 = COPY [[COPY134]] ; DAGISEL-GFX10-NEXT: $vgpr30 = COPY [[COPY133]] ; DAGISEL-GFX10-NEXT: $vgpr31 = COPY [[COPY132]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $vgpr31 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 528, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 diff --git a/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll b/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll index 1acbb0911828..fbf2ee1145ae 100644 --- a/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll +++ b/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll @@ -60,7 +60,6 @@ define amdgpu_kernel void @test_kernel(i32 %val) #0 { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_or_saveexec_b64 s[34:35], -1 diff --git a/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll b/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll index cbdc7bb45634..69971bca2738 100644 --- a/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll +++ b/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll @@ -27,7 +27,7 @@ define internal fastcc void @csr_vgpr_spill_fp_callee() #0 { ; CHECK-LABEL: csr_vgpr_spill_fp_callee: ; CHECK: ; %bb.0: ; %bb ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_mov_b32 s24, s33 +; CHECK-NEXT: s_mov_b32 s18, s33 ; CHECK-NEXT: s_mov_b32 s33, s32 ; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill @@ -43,7 +43,6 @@ define internal fastcc void @csr_vgpr_spill_fp_callee() #0 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: ;;#ASMSTART ; CHECK-NEXT: ; clobber csr v40 @@ -55,7 +54,7 @@ define internal fastcc void @csr_vgpr_spill_fp_callee() #0 { ; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload ; CHECK-NEXT: s_mov_b64 exec, s[4:5] ; CHECK-NEXT: s_add_i32 s32, s32, 0xfffffc00 -; CHECK-NEXT: s_mov_b32 s33, s24 +; CHECK-NEXT: s_mov_b32 s33, s18 ; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: s_setpc_b64 s[30:31] bb: @@ -88,7 +87,6 @@ define amdgpu_kernel void @kernel_call() { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm bb: @@ -148,7 +146,6 @@ define amdgpu_kernel void @kernel_tailcall() { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm bb: @@ -173,7 +170,7 @@ define hidden i32 @caller_save_vgpr_spill_fp_tail_call() #0 { ; CHECK-LABEL: caller_save_vgpr_spill_fp_tail_call: ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_mov_b32 s24, s33 +; CHECK-NEXT: s_mov_b32 s18, s33 ; CHECK-NEXT: s_mov_b32 s33, s32 ; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s33 ; 4-byte Folded Spill @@ -188,7 +185,6 @@ define hidden i32 @caller_save_vgpr_spill_fp_tail_call() #0 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: v_readlane_b32 s31, v1, 1 ; CHECK-NEXT: v_readlane_b32 s30, v1, 0 @@ -196,7 +192,7 @@ define hidden i32 @caller_save_vgpr_spill_fp_tail_call() #0 { ; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload ; CHECK-NEXT: s_mov_b64 exec, s[4:5] ; CHECK-NEXT: s_add_i32 s32, s32, 0xfffffc00 -; CHECK-NEXT: s_mov_b32 s33, s24 +; CHECK-NEXT: s_mov_b32 s33, s18 ; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: s_setpc_b64 s[30:31] entry: @@ -208,7 +204,7 @@ define hidden i32 @caller_save_vgpr_spill_fp() #0 { ; CHECK-LABEL: caller_save_vgpr_spill_fp: ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_mov_b32 s25, s33 +; CHECK-NEXT: s_mov_b32 s19, s33 ; CHECK-NEXT: s_mov_b32 s33, s32 ; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s33 ; 4-byte Folded Spill @@ -223,7 +219,6 @@ define hidden i32 @caller_save_vgpr_spill_fp() #0 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: v_readlane_b32 s31, v2, 1 ; CHECK-NEXT: v_readlane_b32 s30, v2, 0 @@ -231,7 +226,7 @@ define hidden i32 @caller_save_vgpr_spill_fp() #0 { ; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s33 ; 4-byte Folded Reload ; CHECK-NEXT: s_mov_b64 exec, s[4:5] ; CHECK-NEXT: s_add_i32 s32, s32, 0xfffffc00 -; CHECK-NEXT: s_mov_b32 s33, s25 +; CHECK-NEXT: s_mov_b32 s33, s19 ; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: s_setpc_b64 s[30:31] entry: @@ -263,7 +258,6 @@ define protected amdgpu_kernel void @kernel() { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm entry: diff --git a/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll b/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll index 34e67d0993fb..9999cb9173b5 100644 --- a/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll +++ b/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll @@ -32,7 +32,6 @@ define hidden void @_ZL3barv() #0 !dbg !1644 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: .Ltmp1: diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll b/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll index 764f4942cbd0..f523b4a2495f 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll @@ -16,7 +16,7 @@ define void @spill_sgpr_with_no_lower_vgpr_available() #0 { ; GCN-LABEL: spill_sgpr_with_no_lower_vgpr_available: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s24, s33 +; GCN-NEXT: s_mov_b32 s18, s33 ; GCN-NEXT: s_mov_b32 s33, s32 ; GCN-NEXT: s_or_saveexec_b64 s[16:17], -1 ; GCN-NEXT: buffer_store_dword v255, off, s[0:3], s33 offset:448 ; 4-byte Folded Spill @@ -150,7 +150,6 @@ define void @spill_sgpr_with_no_lower_vgpr_available() #0 { ; GCN-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-NEXT: s_mov_b64 s[2:3], s[22:23] -; GCN-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-NEXT: v_readlane_b32 s31, v255, 1 @@ -270,7 +269,7 @@ define void @spill_sgpr_with_no_lower_vgpr_available() #0 { ; GCN-NEXT: buffer_load_dword v255, off, s[0:3], s33 offset:448 ; 4-byte Folded Reload ; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: s_add_i32 s32, s32, 0xffff8c00 -; GCN-NEXT: s_mov_b32 s33, s24 +; GCN-NEXT: s_mov_b32 s33, s18 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] %alloca = alloca i32, align 4, addrspace(5) @@ -311,7 +310,7 @@ define void @spill_to_lowest_available_vgpr() #0 { ; GCN-LABEL: spill_to_lowest_available_vgpr: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s24, s33 +; GCN-NEXT: s_mov_b32 s18, s33 ; GCN-NEXT: s_mov_b32 s33, s32 ; GCN-NEXT: s_or_saveexec_b64 s[16:17], -1 ; GCN-NEXT: buffer_store_dword v254, off, s[0:3], s33 offset:444 ; 4-byte Folded Spill @@ -444,7 +443,6 @@ define void @spill_to_lowest_available_vgpr() #0 { ; GCN-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-NEXT: s_mov_b64 s[2:3], s[22:23] -; GCN-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-NEXT: v_readlane_b32 s31, v254, 1 @@ -563,7 +561,7 @@ define void @spill_to_lowest_available_vgpr() #0 { ; GCN-NEXT: buffer_load_dword v254, off, s[0:3], s33 offset:444 ; 4-byte Folded Reload ; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: s_add_i32 s32, s32, 0xffff8c00 -; GCN-NEXT: s_mov_b32 s33, s24 +; GCN-NEXT: s_mov_b32 s33, s18 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] %alloca = alloca i32, align 4, addrspace(5) @@ -1530,7 +1528,7 @@ define void @spill_sgpr_no_free_vgpr_ipra() #0 { ; GCN-LABEL: spill_sgpr_no_free_vgpr_ipra: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s24, s33 +; GCN-NEXT: s_mov_b32 s18, s33 ; GCN-NEXT: s_mov_b32 s33, s32 ; GCN-NEXT: s_add_i32 s32, s32, 0x7400 ; GCN-NEXT: buffer_store_dword v40, off, s[0:3], s33 offset:444 ; 4-byte Folded Spill @@ -1668,7 +1666,6 @@ define void @spill_sgpr_no_free_vgpr_ipra() #0 { ; GCN-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-NEXT: s_mov_b64 s[2:3], s[22:23] -; GCN-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-NEXT: s_mov_b64 s[4:5], exec ; GCN-NEXT: s_mov_b64 exec, 1 @@ -1801,7 +1798,7 @@ define void @spill_sgpr_no_free_vgpr_ipra() #0 { ; GCN-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:440 ; 4-byte Folded Reload ; GCN-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:444 ; 4-byte Folded Reload ; GCN-NEXT: s_add_i32 s32, s32, 0xffff8c00 -; GCN-NEXT: s_mov_b32 s33, s24 +; GCN-NEXT: s_mov_b32 s33, s18 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] call void @child_function_ipra() diff --git a/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll b/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll index b8bc01e0b879..c6a599094fe4 100644 --- a/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll +++ b/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll @@ -916,13 +916,13 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-O0-LABEL: kernel_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-O0: ; %bb.0: ; WAVE32-O0-NEXT: s_mov_b32 s32, 0x1200 -; WAVE32-O0-NEXT: s_getpc_b64 s[24:25] -; WAVE32-O0-NEXT: s_mov_b32 s24, s0 -; WAVE32-O0-NEXT: s_load_dwordx4 s[24:27], s[24:25], 0x0 +; WAVE32-O0-NEXT: s_getpc_b64 s[20:21] +; WAVE32-O0-NEXT: s_mov_b32 s20, s0 +; WAVE32-O0-NEXT: s_load_dwordx4 s[20:23], s[20:21], 0x0 ; WAVE32-O0-NEXT: s_waitcnt lgkmcnt(0) -; WAVE32-O0-NEXT: s_bitset0_b32 s27, 21 -; WAVE32-O0-NEXT: s_add_u32 s24, s24, s9 -; WAVE32-O0-NEXT: s_addc_u32 s25, s25, 0 +; WAVE32-O0-NEXT: s_bitset0_b32 s23, 21 +; WAVE32-O0-NEXT: s_add_u32 s20, s20, s9 +; WAVE32-O0-NEXT: s_addc_u32 s21, s21, 0 ; WAVE32-O0-NEXT: ; implicit-def: $vgpr3 : SGPR spill to VGPR lane ; WAVE32-O0-NEXT: s_mov_b32 s14, s8 ; WAVE32-O0-NEXT: s_mov_b32 s13, s7 @@ -934,17 +934,17 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-O0-NEXT: v_writelane_b32 v3, s0, 0 ; WAVE32-O0-NEXT: s_lshr_b32 s0, s0, 5 ; WAVE32-O0-NEXT: v_writelane_b32 v3, s0, 1 -; WAVE32-O0-NEXT: s_or_saveexec_b32 s20, -1 -; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[24:27], 0 offset:128 ; 4-byte Folded Spill -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s20 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s19, -1 +; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[20:23], 0 offset:128 ; 4-byte Folded Spill +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s19 ; WAVE32-O0-NEXT: v_mov_b32_e32 v3, 42 -; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[24:27], 0 +; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[20:23], 0 ; WAVE32-O0-NEXT: s_waitcnt_vscnt null, 0x0 -; WAVE32-O0-NEXT: s_mov_b64 s[0:1], s[24:25] -; WAVE32-O0-NEXT: s_mov_b64 s[2:3], s[26:27] +; WAVE32-O0-NEXT: s_mov_b64 s[0:1], s[20:21] +; WAVE32-O0-NEXT: s_mov_b64 s[2:3], s[22:23] ; WAVE32-O0-NEXT: s_mov_b32 s6, s32 ; WAVE32-O0-NEXT: v_mov_b32_e32 v3, 17 -; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[24:27], s6 offset:4 +; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[20:23], s6 offset:4 ; WAVE32-O0-NEXT: s_mov_b32 s6, stack_passed_argument@abs32@hi ; WAVE32-O0-NEXT: s_mov_b32 s16, stack_passed_argument@abs32@lo ; WAVE32-O0-NEXT: ; kill: def $sgpr16 killed $sgpr16 def $sgpr16_sgpr17 @@ -1018,11 +1018,10 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE32-O0-NEXT: v_mov_b32_e32 v30, s18 -; WAVE32-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] -; WAVE32-O0-NEXT: s_or_saveexec_b32 s20, -1 -; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:128 ; 4-byte Folded Reload -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s20 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s19, -1 +; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:128 ; 4-byte Folded Reload +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s19 ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE32-O0-NEXT: v_readlane_b32 s1, v0, 1 ; WAVE32-O0-NEXT: v_readlane_b32 s0, v0, 0 @@ -1137,7 +1136,6 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE64-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE64-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE64-O0-NEXT: v_mov_b32_e32 v30, s18 -; WAVE64-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE64-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE64-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 ; WAVE64-O0-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:128 ; 4-byte Folded Reload @@ -1155,13 +1153,13 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-WWM-PREALLOC-LABEL: kernel_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-WWM-PREALLOC: ; %bb.0: ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s32, 0x1200 -; WAVE32-WWM-PREALLOC-NEXT: s_getpc_b64 s[24:25] -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s24, s0 -; WAVE32-WWM-PREALLOC-NEXT: s_load_dwordx4 s[24:27], s[24:25], 0x0 +; WAVE32-WWM-PREALLOC-NEXT: s_getpc_b64 s[20:21] +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s20, s0 +; WAVE32-WWM-PREALLOC-NEXT: s_load_dwordx4 s[20:23], s[20:21], 0x0 ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt lgkmcnt(0) -; WAVE32-WWM-PREALLOC-NEXT: s_bitset0_b32 s27, 21 -; WAVE32-WWM-PREALLOC-NEXT: s_add_u32 s24, s24, s9 -; WAVE32-WWM-PREALLOC-NEXT: s_addc_u32 s25, s25, 0 +; WAVE32-WWM-PREALLOC-NEXT: s_bitset0_b32 s23, 21 +; WAVE32-WWM-PREALLOC-NEXT: s_add_u32 s20, s20, s9 +; WAVE32-WWM-PREALLOC-NEXT: s_addc_u32 s21, s21, 0 ; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $vgpr32 : SGPR spill to VGPR lane ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s14, s8 ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s13, s7 @@ -1174,13 +1172,13 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-WWM-PREALLOC-NEXT: s_lshr_b32 s0, s0, 5 ; WAVE32-WWM-PREALLOC-NEXT: v_writelane_b32 v32, s0, 1 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v3, 42 -; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[24:27], 0 +; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[20:23], 0 ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt_vscnt null, 0x0 -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[0:1], s[24:25] -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[2:3], s[26:27] +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[0:1], s[20:21] +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[2:3], s[22:23] ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s6, s32 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v3, 17 -; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[24:27], s6 offset:4 +; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[20:23], s6 offset:4 ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s6, stack_passed_argument@abs32@hi ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s16, stack_passed_argument@abs32@lo ; WAVE32-WWM-PREALLOC-NEXT: ; kill: def $sgpr16 killed $sgpr16 def $sgpr16_sgpr17 @@ -1254,7 +1252,6 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v30, s18 -; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-WWM-PREALLOC-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s1, v32, 1 ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s0, v32, 0 @@ -1347,7 +1344,7 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-LABEL: func_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-O0: ; %bb.0: ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; WAVE32-O0-NEXT: s_mov_b32 s26, s33 +; WAVE32-O0-NEXT: s_mov_b32 s25, s33 ; WAVE32-O0-NEXT: s_mov_b32 s33, s32 ; WAVE32-O0-NEXT: s_xor_saveexec_b32 s16, -1 ; WAVE32-O0-NEXT: buffer_store_dword v32, off, s[0:3], s33 offset:128 ; 4-byte Folded Spill @@ -1361,9 +1358,9 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-NEXT: v_writelane_b32 v0, s16, 0 ; WAVE32-O0-NEXT: s_lshr_b32 s16, s16, 5 ; WAVE32-O0-NEXT: v_writelane_b32 v0, s16, 1 -; WAVE32-O0-NEXT: s_or_saveexec_b32 s25, -1 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s24, -1 ; WAVE32-O0-NEXT: buffer_store_dword v0, off, s[0:3], s33 offset:132 ; 4-byte Folded Spill -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s25 +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s24 ; WAVE32-O0-NEXT: v_mov_b32_e32 v0, 42 ; WAVE32-O0-NEXT: buffer_store_dword v0, off, s[0:3], s33 ; WAVE32-O0-NEXT: s_waitcnt_vscnt null, 0x0 @@ -1440,11 +1437,10 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE32-O0-NEXT: v_mov_b32_e32 v30, s18 -; WAVE32-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] -; WAVE32-O0-NEXT: s_or_saveexec_b32 s25, -1 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s24, -1 ; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s25 +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s24 ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE32-O0-NEXT: v_readlane_b32 s5, v0, 1 ; WAVE32-O0-NEXT: v_readlane_b32 s4, v0, 0 @@ -1460,14 +1456,14 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:136 ; 4-byte Folded Reload ; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s4 ; WAVE32-O0-NEXT: s_add_i32 s32, s32, 0xffffee00 -; WAVE32-O0-NEXT: s_mov_b32 s33, s26 +; WAVE32-O0-NEXT: s_mov_b32 s33, s25 ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE32-O0-NEXT: s_setpc_b64 s[30:31] ; ; WAVE64-O0-LABEL: func_stacksave_stackrestore_call_with_stack_objects: ; WAVE64-O0: ; %bb.0: ; WAVE64-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; WAVE64-O0-NEXT: s_mov_b32 s28, s33 +; WAVE64-O0-NEXT: s_mov_b32 s19, s33 ; WAVE64-O0-NEXT: s_mov_b32 s33, s32 ; WAVE64-O0-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; WAVE64-O0-NEXT: buffer_store_dword v32, off, s[0:3], s33 offset:128 ; 4-byte Folded Spill @@ -1560,7 +1556,6 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE64-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE64-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE64-O0-NEXT: v_mov_b32_e32 v30, s18 -; WAVE64-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE64-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE64-O0-NEXT: s_or_saveexec_b64 s[26:27], -1 ; WAVE64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload @@ -1580,14 +1575,14 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:136 ; 4-byte Folded Reload ; WAVE64-O0-NEXT: s_mov_b64 exec, s[4:5] ; WAVE64-O0-NEXT: s_add_i32 s32, s32, 0xffffdc00 -; WAVE64-O0-NEXT: s_mov_b32 s33, s28 +; WAVE64-O0-NEXT: s_mov_b32 s33, s19 ; WAVE64-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE64-O0-NEXT: s_setpc_b64 s[30:31] ; ; WAVE32-WWM-PREALLOC-LABEL: func_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-WWM-PREALLOC: ; %bb.0: ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s25, s33 +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s24, s33 ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s33, s32 ; WAVE32-WWM-PREALLOC-NEXT: s_xor_saveexec_b32 s16, -1 ; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v33, off, s[0:3], s33 offset:128 ; 4-byte Folded Spill @@ -1677,7 +1672,6 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v30, s18 -; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-WWM-PREALLOC-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s5, v32, 1 ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s4, v32, 0 @@ -1693,7 +1687,7 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-WWM-PREALLOC-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 exec_lo, s4 ; WAVE32-WWM-PREALLOC-NEXT: s_add_i32 s32, s32, 0xffffee00 -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s33, s25 +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s33, s24 ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt vmcnt(0) ; WAVE32-WWM-PREALLOC-NEXT: s_setpc_b64 s[30:31] %alloca = alloca [32 x i32], addrspace(5) diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll index d2364a61ed68..bfc249e9081d 100644 --- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll +++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll @@ -233,10 +233,10 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun ; SI-NEXT: bb.1.Flow: ; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %49:vgpr_32, %bb.0, %4, %bb.9 - ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %51:vgpr_32, %bb.9 - ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %53:vgpr_32, %bb.9 - ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %55:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %47:vgpr_32, %bb.0, %4, %bb.9 + ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %49:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %51:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %53:vgpr_32, %bb.9 ; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; SI-NEXT: S_BRANCH %bb.2 ; SI-NEXT: {{ $}} @@ -249,8 +249,8 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun ; SI-NEXT: bb.3: ; SI-NEXT: successors: %bb.4(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %57:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 - ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI undef %59:vgpr_32, %bb.4, [[PHI1]], %bb.2 + ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %55:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 + ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI undef %57:vgpr_32, %bb.4, [[PHI1]], %bb.2 ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1 @@ -286,8 +286,8 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun ; SI-NEXT: bb.7: ; SI-NEXT: successors: %bb.8(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %61:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 - ; SI-NEXT: [[PHI7:%[0-9]+]]:vgpr_32 = PHI undef %63:vgpr_32, %bb.8, [[COPY4]], %bb.6 + ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %59:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 + ; SI-NEXT: [[PHI7:%[0-9]+]]:vgpr_32 = PHI undef %61:vgpr_32, %bb.8, [[COPY4]], %bb.6 ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI6]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI6]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1 @@ -356,9 +356,9 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e ; SI-NEXT: bb.1.Flow: ; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %50:vgpr_32, %bb.0, %4, %bb.9 - ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %52:vgpr_32, %bb.9 - ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %54:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %48:vgpr_32, %bb.0, %4, %bb.9 + ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %50:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %52:vgpr_32, %bb.9 ; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; SI-NEXT: S_BRANCH %bb.2 ; SI-NEXT: {{ $}} @@ -371,7 +371,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e ; SI-NEXT: bb.3: ; SI-NEXT: successors: %bb.4(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI3:%[0-9]+]]:vreg_64 = PHI undef %56:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 + ; SI-NEXT: [[PHI3:%[0-9]+]]:vreg_64 = PHI undef %54:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI3]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI3]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1 @@ -407,7 +407,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e ; SI-NEXT: bb.7: ; SI-NEXT: successors: %bb.8(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %58:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 + ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %56:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1 diff --git a/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll b/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll index 37f207fc7a54..4939d52651d9 100644 --- a/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll +++ b/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll @@ -47,7 +47,6 @@ define protected amdgpu_kernel void @kern(ptr %addr) !llvm.amdgcn.lds.kernel.id ; CHECK-NEXT: s_mov_b32 s15, 42 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll b/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll index 3a33194f17c8..7eabe982ff2b 100644 --- a/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll +++ b/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll @@ -101,7 +101,6 @@ define void @test() #0 { ; GCN-O0-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-O0-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-O0-NEXT: s_mov_b64 s[2:3], s[22:23] -; GCN-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-O0-NEXT: s_waitcnt lgkmcnt(0) ; GCN-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-O0-NEXT: s_or_saveexec_b64 s[28:29], -1 diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll index 11f6a2960776..e79cb66dcd77 100644 --- a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll +++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll @@ -406,7 +406,6 @@ define amdgpu_gfx void @strict_wwm_call(ptr addrspace(8) inreg %tmp14, i32 inreg ; GFX9-O0-NEXT: s_mov_b64 s[0:1], s[44:45] ; GFX9-O0-NEXT: s_mov_b64 s[2:3], s[46:47] ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-O0-NEXT: ; implicit-def: $sgpr44_sgpr45 ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[42:43] ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-O0-NEXT: v_add_u32_e64 v1, v1, v2 @@ -633,7 +632,6 @@ define amdgpu_gfx void @strict_wwm_call_i64(ptr addrspace(8) inreg %tmp14, i64 i ; GFX9-O0-NEXT: s_mov_b64 s[2:3], s[38:39] ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v2 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 -; GFX9-O0-NEXT: ; implicit-def: $sgpr36_sgpr37 ; GFX9-O0-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[46:47], -1 diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll index e5cebc1c3183..def51f2b16d3 100644 --- a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll +++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll @@ -413,7 +413,6 @@ define amdgpu_kernel void @call(ptr addrspace(8) inreg %tmp14, i32 inreg %arg) { ; GFX9-O0-NEXT: ; implicit-def: $sgpr15 ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 ; GFX9-O0-NEXT: buffer_load_dword v1, off, s[24:27], 0 ; 4-byte Folded Reload @@ -657,7 +656,6 @@ define amdgpu_kernel void @call_i64(ptr addrspace(8) inreg %tmp14, i64 inreg %ar ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 @@ -1285,7 +1283,6 @@ define amdgpu_kernel void @strict_wwm_call(ptr addrspace(8) inreg %tmp14, i32 in ; GFX9-O0-NEXT: ; implicit-def: $sgpr15 ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 ; GFX9-O0-NEXT: buffer_load_dword v1, off, s[24:27], 0 ; 4-byte Folded Reload @@ -1529,7 +1526,6 @@ define amdgpu_kernel void @strict_wwm_call_i64(ptr addrspace(8) inreg %tmp14, i6 ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 -- GitLab From 1e442ac4c33ac36d33e191c2d18ff594d8a5d11a Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 31 Mar 2024 22:20:57 -0700 Subject: [PATCH 0086/1690] [CostModel] No cost for llvm.allow.{runtime,ubsan}.check() (#86064) These intrinsics will not be lowered to code. RFC: https://discourse.llvm.org/t/rfc-add-llvm-experimental-hot-intrinsic-or-llvm-hot/77641 --- llvm/include/llvm/Analysis/TargetTransformInfoImpl.h | 2 ++ llvm/test/Analysis/CostModel/free-intrinsics-datalayout.ll | 6 ++++++ llvm/test/Analysis/CostModel/free-intrinsics-no_info.ll | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 7c47d3c2338a..63c2ef8912b2 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -714,6 +714,8 @@ public: switch (ICA.getID()) { default: break; + case Intrinsic::allow_runtime_check: + case Intrinsic::allow_ubsan_check: case Intrinsic::annotation: case Intrinsic::assume: case Intrinsic::sideeffect: diff --git a/llvm/test/Analysis/CostModel/free-intrinsics-datalayout.ll b/llvm/test/Analysis/CostModel/free-intrinsics-datalayout.ll index 4ebb1e314ba1..09fbd68d95e9 100644 --- a/llvm/test/Analysis/CostModel/free-intrinsics-datalayout.ll +++ b/llvm/test/Analysis/CostModel/free-intrinsics-datalayout.ll @@ -25,6 +25,8 @@ define i32 @trivially_free() { ; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(i64 1, ptr undef) ; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 true, i1 true, i1 true) ; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a6 = call ptr @llvm.ptr.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) +; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a7 = call i1 @llvm.allow.ubsan.check(i8 123) +; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a8 = call i1 @llvm.allow.runtime.check(metadata !"test_check") ; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.var.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) ; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; @@ -45,6 +47,8 @@ define i32 @trivially_free() { ; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(i64 1, ptr undef) ; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 true, i1 true, i1 true) ; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a6 = call ptr @llvm.ptr.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) +; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a7 = call i1 @llvm.allow.ubsan.check(i8 123) +; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a8 = call i1 @llvm.allow.runtime.check(metadata !"test_check") ; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.var.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) ; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; @@ -65,6 +69,8 @@ define i32 @trivially_free() { call void @llvm.lifetime.end.p0(i64 1, ptr undef) %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 1, i1 1, i1 1) %a6 = call ptr @llvm.ptr.annotation.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) + %a7 = call i1 @llvm.allow.ubsan.check(i8 123) + %a8 = call i1 @llvm.allow.runtime.check(metadata !"test_check") call void @llvm.var.annotation(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) ret i32 undef } diff --git a/llvm/test/Analysis/CostModel/free-intrinsics-no_info.ll b/llvm/test/Analysis/CostModel/free-intrinsics-no_info.ll index 5d47e4894427..3e78c62a6fe2 100644 --- a/llvm/test/Analysis/CostModel/free-intrinsics-no_info.ll +++ b/llvm/test/Analysis/CostModel/free-intrinsics-no_info.ll @@ -24,6 +24,8 @@ define i32 @trivially_free() { ; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 true, i1 true, i1 true) ; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a6 = call ptr @llvm.ptr.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) ; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.var.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) +; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a7 = call i1 @llvm.allow.ubsan.check(i8 123) +; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a8 = call i1 @llvm.allow.runtime.check(metadata !"test_check") ; CHECK-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; ; CHECK-THROUGHPUT-LABEL: 'trivially_free' @@ -44,6 +46,8 @@ define i32 @trivially_free() { ; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 true, i1 true, i1 true) ; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a6 = call ptr @llvm.ptr.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) ; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.var.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) +; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a7 = call i1 @llvm.allow.ubsan.check(i8 123) +; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %a8 = call i1 @llvm.allow.runtime.check(metadata !"test_check") ; CHECK-THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %a0 = call i32 @llvm.annotation.i32(i32 undef, ptr undef, ptr undef, i32 undef) @@ -64,6 +68,8 @@ define i32 @trivially_free() { %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 1, i1 1, i1 1) %a6 = call ptr @llvm.ptr.annotation.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) call void @llvm.var.annotation(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef) + %a7 = call i1 @llvm.allow.ubsan.check(i8 123) + %a8 = call i1 @llvm.allow.runtime.check(metadata !"test_check") ret i32 undef } -- GitLab From 0bc3781649c2d029e61766c0b74ea1f416996105 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 31 Mar 2024 22:47:55 -0700 Subject: [PATCH 0087/1690] [Analysis] Exclude llvm.allow.{runtime,ubsan}.check() from AliasSetTracker (#86065) RFC: https://discourse.llvm.org/t/rfc-add-llvm-experimental-hot-intrinsic-or-llvm-hot/77641 --- llvm/lib/Analysis/AliasSetTracker.cpp | 2 ++ llvm/test/Analysis/AliasSet/intrinsics.ll | 30 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/llvm/lib/Analysis/AliasSetTracker.cpp b/llvm/lib/Analysis/AliasSetTracker.cpp index 59bb40cdba6e..32e545daaf22 100644 --- a/llvm/lib/Analysis/AliasSetTracker.cpp +++ b/llvm/lib/Analysis/AliasSetTracker.cpp @@ -354,6 +354,8 @@ void AliasSetTracker::addUnknown(Instruction *Inst) { default: break; // FIXME: Add lifetime/invariant intrinsics (See: PR30807). + case Intrinsic::allow_runtime_check: + case Intrinsic::allow_ubsan_check: case Intrinsic::assume: case Intrinsic::experimental_noalias_scope_decl: case Intrinsic::sideeffect: diff --git a/llvm/test/Analysis/AliasSet/intrinsics.ll b/llvm/test/Analysis/AliasSet/intrinsics.ll index 678d6d246e63..0dc802ca7e0a 100644 --- a/llvm/test/Analysis/AliasSet/intrinsics.ll +++ b/llvm/test/Analysis/AliasSet/intrinsics.ll @@ -74,6 +74,36 @@ entry: ret void } +; CHECK: Alias sets for function 'test_runtime': +; CHECK: Alias Set Tracker: 2 alias sets for 2 pointer values. +; CHECK: AliasSet[0x{{[0-9a-f]+}}, 1] must alias, Mod Memory locations: (ptr %a, LocationSize::precise(1)) +; CHECK: AliasSet[0x{{[0-9a-f]+}}, 1] must alias, Mod Memory locations: (ptr %b, LocationSize::precise(1)) +define i1 @test_runtime() local_unnamed_addr { +entry: + %a = alloca i8, align 1 + %b = alloca i8, align 1 + store i8 1, ptr %a, align 1 + %allow = call i1 @llvm.allow.runtime.check(metadata !"test_check") + store i8 1, ptr %b, align 1 + ret i1 %allow +} + +; CHECK: Alias sets for function 'test_ubsan': +; CHECK: Alias Set Tracker: 2 alias sets for 2 pointer values. +; CHECK: AliasSet[0x{{[0-9a-f]+}}, 1] must alias, Mod Memory locations: (ptr %a, LocationSize::precise(1)) +; CHECK: AliasSet[0x{{[0-9a-f]+}}, 1] must alias, Mod Memory locations: (ptr %b, LocationSize::precise(1)) +define i1 @test_ubsan() local_unnamed_addr { +entry: + %a = alloca i8, align 1 + %b = alloca i8, align 1 + store i8 1, ptr %a, align 1 + %allow = call i1 @llvm.allow.ubsan.check(i8 7) + store i8 1, ptr %b, align 1 + ret i1 %allow +} + +declare i1 @llvm.allow.ubsan.check(i8) +declare i1 @llvm.allow.runtime.check(metadata) declare void @llvm.assume(i1) declare void @llvm.experimental.guard(i1, ...) declare void @llvm.experimental.noalias.scope.decl(metadata) -- GitLab From 37d6e5b7a555e8c85c3e34803a710725c26857c7 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 31 Mar 2024 22:50:02 -0700 Subject: [PATCH 0088/1690] [memoryssa] Exclude llvm.allow.{runtime,ubsan}.check() (#86066) RFC: https://discourse.llvm.org/t/rfc-add-llvm-experimental-hot-intrinsic-or-llvm-hot/77641 --- llvm/lib/Analysis/MemorySSA.cpp | 4 +++ llvm/test/Analysis/MemorySSA/allow-check.ll | 29 +++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 llvm/test/Analysis/MemorySSA/allow-check.ll diff --git a/llvm/lib/Analysis/MemorySSA.cpp b/llvm/lib/Analysis/MemorySSA.cpp index 82a6c470650c..49450d85d742 100644 --- a/llvm/lib/Analysis/MemorySSA.cpp +++ b/llvm/lib/Analysis/MemorySSA.cpp @@ -292,6 +292,8 @@ instructionClobbersQuery(const MemoryDef *MD, const MemoryLocation &UseLoc, // clobbers where they don't really exist at all. Please see D43269 for // context. switch (II->getIntrinsicID()) { + case Intrinsic::allow_runtime_check: + case Intrinsic::allow_ubsan_check: case Intrinsic::invariant_start: case Intrinsic::invariant_end: case Intrinsic::assume: @@ -1725,6 +1727,8 @@ MemoryUseOrDef *MemorySSA::createNewAccess(Instruction *I, switch (II->getIntrinsicID()) { default: break; + case Intrinsic::allow_runtime_check: + case Intrinsic::allow_ubsan_check: case Intrinsic::assume: case Intrinsic::experimental_noalias_scope_decl: case Intrinsic::pseudoprobe: diff --git a/llvm/test/Analysis/MemorySSA/allow-check.ll b/llvm/test/Analysis/MemorySSA/allow-check.ll new file mode 100644 index 000000000000..dcdad001ca6f --- /dev/null +++ b/llvm/test/Analysis/MemorySSA/allow-check.ll @@ -0,0 +1,29 @@ +; RUN: opt -aa-pipeline=basic-aa -passes='print,verify' -disable-output < %s 2>&1 | FileCheck %s --implicit-check-not=MemoryDef +; +; Ensures that allow.*.check are treated as not reading or writing memory. + +target triple = "aarch64-linux" + +define i1 @test_runtime(ptr %a) local_unnamed_addr { +entry: +; CHECK: 1 = MemoryDef(liveOnEntry) + store i32 4, ptr %a, align 4 + %allow = call i1 @llvm.allow.runtime.check(metadata !"test_check") + %0 = load i32, ptr %a, align 4 +; CHECK: MemoryUse(1) + ret i1 %allow +} + +declare i1 @llvm.allow.runtime.check(metadata) + +define i1 @test_ubsan(ptr %a) local_unnamed_addr { +entry: +; CHECK: 1 = MemoryDef(liveOnEntry) + store i32 4, ptr %a, align 4 + %allow = call i1 @llvm.allow.ubsan.check(i8 7) + %0 = load i32, ptr %a, align 4 +; CHECK: MemoryUse(1) + ret i1 %allow +} + +declare i1 @llvm.allow.ubsan.check(i8) -- GitLab From 289d2cc3f398952a6ed2580530a542ca846f86e8 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 31 Mar 2024 23:09:08 -0700 Subject: [PATCH 0089/1690] [CodeGen] Fix test after #86049 --- llvm/test/CodeGen/Generic/allow-check.ll | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llvm/test/CodeGen/Generic/allow-check.ll b/llvm/test/CodeGen/Generic/allow-check.ll index 7c6cb16eacca..3dc3a648f6f5 100644 --- a/llvm/test/CodeGen/Generic/allow-check.ll +++ b/llvm/test/CodeGen/Generic/allow-check.ll @@ -1,3 +1,6 @@ +; Avoid `!DL->isLittleEndian() && !CLI->enableBigEndian()` missmatch on PPC64BE. +; REQUIRES: host-byteorder-little-endian + ; RUN: llc < %s -O3 -global-isel=0 -fast-isel=0 ; RUN: llc < %s -O3 -global-isel=1 -fast-isel=0 ; RUN: llc < %s -O3 -global-isel=0 -fast-isel=1 -- GitLab From b890c17892c75d1d7671b7c2ba1c927d4d3e09e2 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 31 Mar 2024 23:22:07 -0700 Subject: [PATCH 0090/1690] [CodeGen] Fix test after #86049 --- llvm/test/CodeGen/Generic/allow-check.ll | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llvm/test/CodeGen/Generic/allow-check.ll b/llvm/test/CodeGen/Generic/allow-check.ll index 3dc3a648f6f5..493b7f8950b1 100644 --- a/llvm/test/CodeGen/Generic/allow-check.ll +++ b/llvm/test/CodeGen/Generic/allow-check.ll @@ -1,6 +1,9 @@ ; Avoid `!DL->isLittleEndian() && !CLI->enableBigEndian()` missmatch on PPC64BE. ; REQUIRES: host-byteorder-little-endian +; Does not support -global-isel=1. +; UNSUPPORTED: target=nvptx{{.*}} + ; RUN: llc < %s -O3 -global-isel=0 -fast-isel=0 ; RUN: llc < %s -O3 -global-isel=1 -fast-isel=0 ; RUN: llc < %s -O3 -global-isel=0 -fast-isel=1 -- GitLab From c4df57da1d7ede73c11a9ae60f4d4a2ce5132c56 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 31 Mar 2024 23:39:33 -0700 Subject: [PATCH 0091/1690] [CodeGen] llvm.allow.{runtime,ubsan}.check() in FastISel Follow up to #86049. clang-armv8-quick build bot can trigger this branch. --- llvm/lib/CodeGen/SelectionDAG/FastISel.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp index dae827883b29..aefedd060f89 100644 --- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1462,8 +1462,13 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) { llvm_unreachable("llvm.is.constant.* should have been lowered already"); case Intrinsic::allow_runtime_check: - case Intrinsic::allow_ubsan_check: - llvm_unreachable("llvm.*.check should have been lowered already"); + case Intrinsic::allow_ubsan_check: { + Register ResultReg = getRegForValue(ConstantInt::getTrue(II->getType())); + if (!ResultReg) + return false; + updateValueMap(II, ResultReg); + return true; + } case Intrinsic::launder_invariant_group: case Intrinsic::strip_invariant_group: -- GitLab From c7954ca312ca7731a7611c63ff41365c28b06cb5 Mon Sep 17 00:00:00 2001 From: Shih-Po Hung Date: Mon, 1 Apr 2024 14:44:10 +0800 Subject: [PATCH 0092/1690] Recommit "[RISCV] Refine cost on Min/Max reduction (#79402)" (#86480) This is recommitted as the test and fix for llvm.vector.reduce.fmaximum/fminimum are covered in #80553 and #80697 --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 43 +++++- .../Analysis/CostModel/RISCV/reduce-max.ll | 140 +++++++++--------- .../Analysis/CostModel/RISCV/reduce-min.ll | 140 +++++++++--------- .../CostModel/RISCV/reduce-scalable-fp.ll | 84 +++++------ .../CostModel/RISCV/reduce-scalable-int.ll | 96 ++++++------ 5 files changed, 266 insertions(+), 237 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 38cdf3c47c64..efcaa65605e0 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1058,13 +1058,42 @@ RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, } // IR Reduction is composed by two vmv and one rvv reduction instruction. - InstructionCost BaseCost = 2; - - if (CostKind == TTI::TCK_CodeSize) - return (LT.first - 1) + BaseCost; - - unsigned VL = getEstimatedVLFor(Ty); - return (LT.first - 1) + BaseCost + Log2_32_Ceil(VL); + unsigned SplitOp; + SmallVector Opcodes; + switch (IID) { + default: + llvm_unreachable("Unsupported intrinsic"); + case Intrinsic::smax: + SplitOp = RISCV::VMAX_VV; + Opcodes = {RISCV::VMV_S_X, RISCV::VREDMAX_VS, RISCV::VMV_X_S}; + break; + case Intrinsic::smin: + SplitOp = RISCV::VMIN_VV; + Opcodes = {RISCV::VMV_S_X, RISCV::VREDMIN_VS, RISCV::VMV_X_S}; + break; + case Intrinsic::umax: + SplitOp = RISCV::VMAXU_VV; + Opcodes = {RISCV::VMV_S_X, RISCV::VREDMAXU_VS, RISCV::VMV_X_S}; + break; + case Intrinsic::umin: + SplitOp = RISCV::VMINU_VV; + Opcodes = {RISCV::VMV_S_X, RISCV::VREDMINU_VS, RISCV::VMV_X_S}; + break; + case Intrinsic::maxnum: + SplitOp = RISCV::VFMAX_VV; + Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDMAX_VS, RISCV::VFMV_F_S}; + break; + case Intrinsic::minnum: + SplitOp = RISCV::VFMIN_VV; + Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDMIN_VS, RISCV::VFMV_F_S}; + break; + } + // Add a cost for data larger than LMUL8 + InstructionCost SplitCost = + (LT.first > 1) ? (LT.first - 1) * + getRISCVInstructionCost(SplitOp, LT.second, CostKind) + : 0; + return SplitCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind); } InstructionCost diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll index f3436b54df90..533a77dbd9f7 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll @@ -51,14 +51,14 @@ define i32 @reduce_umax_i8(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_umax_i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.umax.v1i8(<1 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i8 @llvm.vector.reduce.umax.v1i8(<1 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i8 @llvm.vector.reduce.umax.v1i8(<1 x i8> undef) @@ -85,14 +85,14 @@ define i32 @reduce_umax_i16(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_umax_i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.umax.v1i16(<1 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i16 @llvm.vector.reduce.umax.v128i16(<128 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i16 @llvm.vector.reduce.umax.v1i16(<1 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i16 @llvm.vector.reduce.umax.v128i16(<128 x i16> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i16 @llvm.vector.reduce.umax.v1i16(<1 x i16> undef) @@ -115,18 +115,18 @@ define i32 @reduce_umax_i32(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i32 @llvm.vector.reduce.umax.v64i32(<64 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call i32 @llvm.vector.reduce.umax.v128i32(<128 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i32 @llvm.vector.reduce.umax.v128i32(<128 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_umax_i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.umax.v1i32(<1 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i32 @llvm.vector.reduce.umax.v64i32(<64 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i32 @llvm.vector.reduce.umax.v128i32(<128 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i32 @llvm.vector.reduce.umax.v1i32(<1 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i32 @llvm.vector.reduce.umax.v64i32(<64 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V128 = call i32 @llvm.vector.reduce.umax.v128i32(<128 x i32> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i32 @llvm.vector.reduce.umax.v1i32(<1 x i32> undef) @@ -148,19 +148,19 @@ define i32 @reduce_umax_i64(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i64 @llvm.vector.reduce.umax.v32i64(<32 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call i64 @llvm.vector.reduce.umax.v64i64(<64 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V128 = call i64 @llvm.vector.reduce.umax.v128i64(<128 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V64 = call i64 @llvm.vector.reduce.umax.v64i64(<64 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V128 = call i64 @llvm.vector.reduce.umax.v128i64(<128 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_umax_i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i64 @llvm.vector.reduce.umax.v32i64(<32 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i64 @llvm.vector.reduce.umax.v64i64(<64 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V128 = call i64 @llvm.vector.reduce.umax.v128i64(<128 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i64 @llvm.vector.reduce.umax.v32i64(<32 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i64 @llvm.vector.reduce.umax.v64i64(<64 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V128 = call i64 @llvm.vector.reduce.umax.v128i64(<128 x i64> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef) @@ -221,14 +221,14 @@ define i32 @reduce_smax_i8(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_smax_i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.smax.v1i8(<1 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i8 @llvm.vector.reduce.smax.v1i8(<1 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i8 @llvm.vector.reduce.smax.v1i8(<1 x i8> undef) @@ -255,14 +255,14 @@ define i32 @reduce_smax_i16(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_smax_i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.smax.v1i16(<1 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i16 @llvm.vector.reduce.smax.v128i16(<128 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i16 @llvm.vector.reduce.smax.v1i16(<1 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i16 @llvm.vector.reduce.smax.v128i16(<128 x i16> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i16 @llvm.vector.reduce.smax.v1i16(<1 x i16> undef) @@ -285,18 +285,18 @@ define i32 @reduce_smax_i32(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i32 @llvm.vector.reduce.smax.v64i32(<64 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call i32 @llvm.vector.reduce.smax.v128i32(<128 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i32 @llvm.vector.reduce.smax.v128i32(<128 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_smax_i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.smax.v1i32(<1 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i32 @llvm.vector.reduce.smax.v64i32(<64 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i32 @llvm.vector.reduce.smax.v128i32(<128 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i32 @llvm.vector.reduce.smax.v1i32(<1 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i32 @llvm.vector.reduce.smax.v64i32(<64 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V128 = call i32 @llvm.vector.reduce.smax.v128i32(<128 x i32> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i32 @llvm.vector.reduce.smax.v1i32(<1 x i32> undef) @@ -318,19 +318,19 @@ define i32 @reduce_smax_i64(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i64 @llvm.vector.reduce.smax.v32i64(<32 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call i64 @llvm.vector.reduce.smax.v64i64(<64 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V128 = call i64 @llvm.vector.reduce.smax.v128i64(<128 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V64 = call i64 @llvm.vector.reduce.smax.v64i64(<64 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V128 = call i64 @llvm.vector.reduce.smax.v128i64(<128 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_smax_i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i64 @llvm.vector.reduce.smax.v32i64(<32 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i64 @llvm.vector.reduce.smax.v64i64(<64 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V128 = call i64 @llvm.vector.reduce.smax.v128i64(<128 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i64 @llvm.vector.reduce.smax.v32i64(<32 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i64 @llvm.vector.reduce.smax.v64i64(<64 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V128 = call i64 @llvm.vector.reduce.smax.v128i64(<128 x i64> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef) diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll index 1964e35b81b1..ae99118d4d3a 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll @@ -51,14 +51,14 @@ define i32 @reduce_umin_i8(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_umin_i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.umin.v1i8(<1 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i8 @llvm.vector.reduce.umin.v1i8(<1 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i8 @llvm.vector.reduce.umin.v1i8(<1 x i8> undef) @@ -85,14 +85,14 @@ define i32 @reduce_umin_i16(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_umin_i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.umin.v1i16(<1 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i16 @llvm.vector.reduce.umin.v128i16(<128 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i16 @llvm.vector.reduce.umin.v1i16(<1 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i16 @llvm.vector.reduce.umin.v128i16(<128 x i16> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i16 @llvm.vector.reduce.umin.v1i16(<1 x i16> undef) @@ -115,18 +115,18 @@ define i32 @reduce_umin_i32(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i32 @llvm.vector.reduce.umin.v64i32(<64 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call i32 @llvm.vector.reduce.umin.v128i32(<128 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i32 @llvm.vector.reduce.umin.v128i32(<128 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_umin_i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.umin.v1i32(<1 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i32 @llvm.vector.reduce.umin.v64i32(<64 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i32 @llvm.vector.reduce.umin.v128i32(<128 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i32 @llvm.vector.reduce.umin.v1i32(<1 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i32 @llvm.vector.reduce.umin.v64i32(<64 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V128 = call i32 @llvm.vector.reduce.umin.v128i32(<128 x i32> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i32 @llvm.vector.reduce.umin.v1i32(<1 x i32> undef) @@ -148,19 +148,19 @@ define i32 @reduce_umin_i64(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i64 @llvm.vector.reduce.umin.v32i64(<32 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call i64 @llvm.vector.reduce.umin.v64i64(<64 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V128 = call i64 @llvm.vector.reduce.umin.v128i64(<128 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V64 = call i64 @llvm.vector.reduce.umin.v64i64(<64 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V128 = call i64 @llvm.vector.reduce.umin.v128i64(<128 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_umin_i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i64 @llvm.vector.reduce.umin.v32i64(<32 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i64 @llvm.vector.reduce.umin.v64i64(<64 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V128 = call i64 @llvm.vector.reduce.umin.v128i64(<128 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i64 @llvm.vector.reduce.umin.v32i64(<32 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i64 @llvm.vector.reduce.umin.v64i64(<64 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V128 = call i64 @llvm.vector.reduce.umin.v128i64(<128 x i64> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef) @@ -221,14 +221,14 @@ define i32 @reduce_smin_i8(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_smin_i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.smin.v1i8(<1 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i8 @llvm.vector.reduce.smin.v1i8(<1 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i8 @llvm.vector.reduce.smin.v1i8(<1 x i8> undef) @@ -255,14 +255,14 @@ define i32 @reduce_smin_i16(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_smin_i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.smin.v1i16(<1 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i16 @llvm.vector.reduce.smin.v128i16(<128 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i16 @llvm.vector.reduce.smin.v1i16(<1 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i16 @llvm.vector.reduce.smin.v128i16(<128 x i16> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i16 @llvm.vector.reduce.smin.v1i16(<1 x i16> undef) @@ -285,18 +285,18 @@ define i32 @reduce_smin_i32(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i32 @llvm.vector.reduce.smin.v64i32(<64 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call i32 @llvm.vector.reduce.smin.v128i32(<128 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i32 @llvm.vector.reduce.smin.v128i32(<128 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_smin_i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.smin.v1i32(<1 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i32 @llvm.vector.reduce.smin.v64i32(<64 x i32> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i32 @llvm.vector.reduce.smin.v128i32(<128 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i32 @llvm.vector.reduce.smin.v1i32(<1 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i32 @llvm.vector.reduce.smin.v64i32(<64 x i32> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V128 = call i32 @llvm.vector.reduce.smin.v128i32(<128 x i32> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i32 @llvm.vector.reduce.smin.v1i32(<1 x i32> undef) @@ -318,19 +318,19 @@ define i32 @reduce_smin_i64(i32 %arg) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i64 @llvm.vector.reduce.smin.v32i64(<32 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call i64 @llvm.vector.reduce.smin.v64i64(<64 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V128 = call i64 @llvm.vector.reduce.smin.v128i64(<128 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V64 = call i64 @llvm.vector.reduce.smin.v64i64(<64 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V128 = call i64 @llvm.vector.reduce.smin.v128i64(<128 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef ; ; SIZE-LABEL: 'reduce_smin_i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i64 @llvm.vector.reduce.smin.v32i64(<32 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i64 @llvm.vector.reduce.smin.v64i64(<64 x i64> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V128 = call i64 @llvm.vector.reduce.smin.v128i64(<128 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i64 @llvm.vector.reduce.smin.v32i64(<32 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i64 @llvm.vector.reduce.smin.v64i64(<64 x i64> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V128 = call i64 @llvm.vector.reduce.smin.v128i64(<128 x i64> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef ; %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef) diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll index 0dd3e3cf53e1..e42dc889f1ba 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll @@ -456,7 +456,7 @@ define half @vreduce_fmin_nxv1f16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv1f16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv1f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv1f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call half @llvm.vector.reduce.fmin.nxv1f16( %v) @@ -469,7 +469,7 @@ define half @vreduce_fmin_nxv1f16_nonans( %v) #0 { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv1f16_nonans' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan half @llvm.vector.reduce.fmin.nxv1f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan half @llvm.vector.reduce.fmin.nxv1f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call nnan half @llvm.vector.reduce.fmin.nxv1f16( %v) @@ -482,7 +482,7 @@ define half @vreduce_fmin_nxv1f16_nonans_noinfs( %v) #1 { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv1f16_nonans_noinfs' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmin.nxv1f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmin.nxv1f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call nnan ninf half @llvm.vector.reduce.fmin.nxv1f16( %v) @@ -497,7 +497,7 @@ define half @vreduce_fmin_nxv2f16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv2f16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv2f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv2f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call half @llvm.vector.reduce.fmin.nxv2f16( %v) @@ -512,7 +512,7 @@ define half @vreduce_fmin_nxv4f16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv4f16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv4f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv4f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call half @llvm.vector.reduce.fmin.nxv4f16( %v) @@ -523,11 +523,11 @@ declare half @llvm.vector.reduce.fmin.nxv64f16() define half @vreduce_fmin_nxv64f16( %v) { ; CHECK-LABEL: 'vreduce_fmin_nxv64f16' -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv64f16( %v) +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv64f16( %v) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv64f16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv64f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv64f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call half @llvm.vector.reduce.fmin.nxv64f16( %v) @@ -542,7 +542,7 @@ define float @vreduce_fmin_nxv1f32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv1f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv1f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv1f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call float @llvm.vector.reduce.fmin.nxv1f32( %v) @@ -555,7 +555,7 @@ define float @vreduce_fmin_nxv1f32_nonans( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv1f32_nonans' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan float @llvm.vector.reduce.fmin.nxv1f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan float @llvm.vector.reduce.fmin.nxv1f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call nnan float @llvm.vector.reduce.fmin.nxv1f32( %v) @@ -568,7 +568,7 @@ define float @vreduce_fmin_nxv1f32_nonans_noinfs( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv1f32_nonans_noinfs' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmin.nxv1f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmin.nxv1f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call nnan ninf float @llvm.vector.reduce.fmin.nxv1f32( %v) @@ -583,7 +583,7 @@ define float @vreduce_fmin_nxv2f32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv2f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv2f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv2f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call float @llvm.vector.reduce.fmin.nxv2f32( %v) @@ -598,7 +598,7 @@ define float @vreduce_fmin_nxv4f32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv4f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv4f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv4f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call float @llvm.vector.reduce.fmin.nxv4f32( %v) @@ -609,11 +609,11 @@ declare float @llvm.vector.reduce.fmin.nxv32f32() define float @vreduce_fmin_nxv32f32( %v) { ; CHECK-LABEL: 'vreduce_fmin_nxv32f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv32f32( %v) +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv32f32( %v) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv32f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv32f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv32f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call float @llvm.vector.reduce.fmin.nxv32f32( %v) @@ -628,7 +628,7 @@ define double @vreduce_fmin_nxv1f64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv1f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv1f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv1f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call double @llvm.vector.reduce.fmin.nxv1f64( %v) @@ -641,7 +641,7 @@ define double @vreduce_fmin_nxv1f64_nonans( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv1f64_nonans' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan double @llvm.vector.reduce.fmin.nxv1f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan double @llvm.vector.reduce.fmin.nxv1f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call nnan double @llvm.vector.reduce.fmin.nxv1f64( %v) @@ -654,7 +654,7 @@ define double @vreduce_fmin_nxv1f64_nonans_noinfs( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv1f64_nonans_noinfs' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmin.nxv1f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmin.nxv1f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call nnan ninf double @llvm.vector.reduce.fmin.nxv1f64( %v) @@ -669,7 +669,7 @@ define double @vreduce_fmin_nxv2f64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv2f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv2f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv2f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call double @llvm.vector.reduce.fmin.nxv2f64( %v) @@ -684,7 +684,7 @@ define double @vreduce_fmin_nxv4f64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv4f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv4f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv4f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call double @llvm.vector.reduce.fmin.nxv4f64( %v) @@ -695,11 +695,11 @@ declare double @llvm.vector.reduce.fmin.nxv16f64() define double @vreduce_fmin_nxv16f64( %v) { ; CHECK-LABEL: 'vreduce_fmin_nxv16f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv16f64( %v) +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv16f64( %v) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmin_nxv16f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv16f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv16f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call double @llvm.vector.reduce.fmin.nxv16f64( %v) @@ -714,7 +714,7 @@ define half @vreduce_fmax_nxv1f16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv1f16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv1f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv1f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call half @llvm.vector.reduce.fmax.nxv1f16( %v) @@ -727,7 +727,7 @@ define half @vreduce_fmax_nxv1f16_nonans( %v) #0 { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv1f16_nonans' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan half @llvm.vector.reduce.fmax.nxv1f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan half @llvm.vector.reduce.fmax.nxv1f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call nnan half @llvm.vector.reduce.fmax.nxv1f16( %v) @@ -740,7 +740,7 @@ define half @vreduce_fmax_nxv1f16_nonans_noinfs( %v) #1 { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv1f16_nonans_noinfs' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmax.nxv1f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmax.nxv1f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call nnan ninf half @llvm.vector.reduce.fmax.nxv1f16( %v) @@ -755,7 +755,7 @@ define half @vreduce_fmax_nxv2f16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv2f16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv2f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv2f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call half @llvm.vector.reduce.fmax.nxv2f16( %v) @@ -770,7 +770,7 @@ define half @vreduce_fmax_nxv4f16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv4f16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv4f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv4f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call half @llvm.vector.reduce.fmax.nxv4f16( %v) @@ -781,11 +781,11 @@ declare half @llvm.vector.reduce.fmax.nxv64f16() define half @vreduce_fmax_nxv64f16( %v) { ; CHECK-LABEL: 'vreduce_fmax_nxv64f16' -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv64f16( %v) +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv64f16( %v) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret half %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv64f16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv64f16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv64f16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret half %red ; %red = call half @llvm.vector.reduce.fmax.nxv64f16( %v) @@ -800,7 +800,7 @@ define float @vreduce_fmax_nxv1f32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv1f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv1f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv1f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call float @llvm.vector.reduce.fmax.nxv1f32( %v) @@ -813,7 +813,7 @@ define float @vreduce_fmax_nxv1f32_nonans( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv1f32_nonans' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan float @llvm.vector.reduce.fmax.nxv1f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan float @llvm.vector.reduce.fmax.nxv1f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call nnan float @llvm.vector.reduce.fmax.nxv1f32( %v) @@ -826,7 +826,7 @@ define float @vreduce_fmax_nxv1f32_nonans_noinfs( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv1f32_nonans_noinfs' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmax.nxv1f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmax.nxv1f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call nnan ninf float @llvm.vector.reduce.fmax.nxv1f32( %v) @@ -841,7 +841,7 @@ define float @vreduce_fmax_nxv2f32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv2f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv2f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv2f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call float @llvm.vector.reduce.fmax.nxv2f32( %v) @@ -856,7 +856,7 @@ define float @vreduce_fmax_nxv4f32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv4f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv4f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv4f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call float @llvm.vector.reduce.fmax.nxv4f32( %v) @@ -867,11 +867,11 @@ declare float @llvm.vector.reduce.fmax.nxv32f32() define float @vreduce_fmax_nxv32f32( %v) { ; CHECK-LABEL: 'vreduce_fmax_nxv32f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv32f32( %v) +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv32f32( %v) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv32f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv32f32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv32f32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float %red ; %red = call float @llvm.vector.reduce.fmax.nxv32f32( %v) @@ -886,7 +886,7 @@ define double @vreduce_fmax_nxv1f64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv1f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv1f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv1f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call double @llvm.vector.reduce.fmax.nxv1f64( %v) @@ -899,7 +899,7 @@ define double @vreduce_fmax_nxv1f64_nonans( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv1f64_nonans' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan double @llvm.vector.reduce.fmax.nxv1f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan double @llvm.vector.reduce.fmax.nxv1f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call nnan double @llvm.vector.reduce.fmax.nxv1f64( %v) @@ -912,7 +912,7 @@ define double @vreduce_fmax_nxv1f64_nonans_noinfs( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv1f64_nonans_noinfs' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmax.nxv1f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmax.nxv1f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call nnan ninf double @llvm.vector.reduce.fmax.nxv1f64( %v) @@ -927,7 +927,7 @@ define double @vreduce_fmax_nxv2f64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv2f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv2f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv2f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call double @llvm.vector.reduce.fmax.nxv2f64( %v) @@ -942,7 +942,7 @@ define double @vreduce_fmax_nxv4f64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv4f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv4f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv4f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call double @llvm.vector.reduce.fmax.nxv4f64( %v) @@ -953,11 +953,11 @@ declare double @llvm.vector.reduce.fmax.nxv16f64() define double @vreduce_fmax_nxv16f64( %v) { ; CHECK-LABEL: 'vreduce_fmax_nxv16f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv16f64( %v) +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv16f64( %v) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; ; SIZE-LABEL: 'vreduce_fmax_nxv16f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv16f64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv16f64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double %red ; %red = call double @llvm.vector.reduce.fmax.nxv16f64( %v) diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll index 30cb32ce4eaf..2807f7526760 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll @@ -27,7 +27,7 @@ define signext i8 @vreduce_umax_nxv1i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv1i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv1i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv1i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.umax.nxv1i8( %v) @@ -42,7 +42,7 @@ define signext i8 @vreduce_smax_nxv1i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv1i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv1i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv1i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.smax.nxv1i8( %v) @@ -57,7 +57,7 @@ define signext i8 @vreduce_umin_nxv1i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv1i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv1i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv1i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.umin.nxv1i8( %v) @@ -72,7 +72,7 @@ define signext i8 @vreduce_smin_nxv1i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv1i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv1i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv1i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.smin.nxv1i8( %v) @@ -147,7 +147,7 @@ define signext i8 @vreduce_umax_nxv2i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv2i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv2i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv2i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.umax.nxv2i8( %v) @@ -162,7 +162,7 @@ define signext i8 @vreduce_smax_nxv2i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv2i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv2i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv2i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.smax.nxv2i8( %v) @@ -177,7 +177,7 @@ define signext i8 @vreduce_umin_nxv2i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv2i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv2i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv2i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.umin.nxv2i8( %v) @@ -192,7 +192,7 @@ define signext i8 @vreduce_smin_nxv2i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv2i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv2i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv2i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.smin.nxv2i8( %v) @@ -267,7 +267,7 @@ define signext i8 @vreduce_umax_nxv4i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv4i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv4i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv4i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.umax.nxv4i8( %v) @@ -282,7 +282,7 @@ define signext i8 @vreduce_smax_nxv4i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv4i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv4i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv4i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.smax.nxv4i8( %v) @@ -297,7 +297,7 @@ define signext i8 @vreduce_umin_nxv4i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv4i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv4i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv4i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.umin.nxv4i8( %v) @@ -312,7 +312,7 @@ define signext i8 @vreduce_smin_nxv4i8( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv4i8' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv4i8( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv4i8( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red ; %red = call i8 @llvm.vector.reduce.smin.nxv4i8( %v) @@ -419,7 +419,7 @@ define signext i16 @vreduce_umax_nxv1i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv1i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv1i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv1i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.umax.nxv1i16( %v) @@ -434,7 +434,7 @@ define signext i16 @vreduce_smax_nxv1i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv1i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv1i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv1i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.smax.nxv1i16( %v) @@ -449,7 +449,7 @@ define signext i16 @vreduce_umin_nxv1i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv1i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv1i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv1i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.umin.nxv1i16( %v) @@ -464,7 +464,7 @@ define signext i16 @vreduce_smin_nxv1i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv1i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv1i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv1i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.smin.nxv1i16( %v) @@ -571,7 +571,7 @@ define signext i16 @vreduce_umax_nxv2i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv2i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv2i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv2i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.umax.nxv2i16( %v) @@ -586,7 +586,7 @@ define signext i16 @vreduce_smax_nxv2i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv2i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv2i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv2i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.smax.nxv2i16( %v) @@ -601,7 +601,7 @@ define signext i16 @vreduce_umin_nxv2i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv2i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv2i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv2i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.umin.nxv2i16( %v) @@ -616,7 +616,7 @@ define signext i16 @vreduce_smin_nxv2i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv2i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv2i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv2i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.smin.nxv2i16( %v) @@ -723,7 +723,7 @@ define signext i16 @vreduce_umax_nxv4i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv4i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv4i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv4i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.umax.nxv4i16( %v) @@ -738,7 +738,7 @@ define signext i16 @vreduce_smax_nxv4i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv4i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv4i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv4i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.smax.nxv4i16( %v) @@ -753,7 +753,7 @@ define signext i16 @vreduce_umin_nxv4i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv4i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv4i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv4i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.umin.nxv4i16( %v) @@ -768,7 +768,7 @@ define signext i16 @vreduce_smin_nxv4i16( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv4i16' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv4i16( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv4i16( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red ; %red = call i16 @llvm.vector.reduce.smin.nxv4i16( %v) @@ -875,7 +875,7 @@ define signext i32 @vreduce_umax_nxv1i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv1i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv1i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv1i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.umax.nxv1i32( %v) @@ -890,7 +890,7 @@ define signext i32 @vreduce_smax_nxv1i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv1i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv1i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv1i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.smax.nxv1i32( %v) @@ -905,7 +905,7 @@ define signext i32 @vreduce_umin_nxv1i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv1i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv1i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv1i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.umin.nxv1i32( %v) @@ -920,7 +920,7 @@ define signext i32 @vreduce_smin_nxv1i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv1i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv1i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv1i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.smin.nxv1i32( %v) @@ -1027,7 +1027,7 @@ define signext i32 @vreduce_umax_nxv2i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv2i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv2i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv2i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.umax.nxv2i32( %v) @@ -1042,7 +1042,7 @@ define signext i32 @vreduce_smax_nxv2i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv2i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv2i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv2i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.smax.nxv2i32( %v) @@ -1057,7 +1057,7 @@ define signext i32 @vreduce_umin_nxv2i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv2i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv2i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv2i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.umin.nxv2i32( %v) @@ -1072,7 +1072,7 @@ define signext i32 @vreduce_smin_nxv2i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv2i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv2i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv2i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.smin.nxv2i32( %v) @@ -1179,7 +1179,7 @@ define signext i32 @vreduce_umax_nxv4i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv4i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv4i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv4i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.umax.nxv4i32( %v) @@ -1194,7 +1194,7 @@ define signext i32 @vreduce_smax_nxv4i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv4i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv4i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv4i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.smax.nxv4i32( %v) @@ -1209,7 +1209,7 @@ define signext i32 @vreduce_umin_nxv4i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv4i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv4i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv4i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.umin.nxv4i32( %v) @@ -1224,7 +1224,7 @@ define signext i32 @vreduce_smin_nxv4i32( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv4i32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv4i32( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv4i32( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red ; %red = call i32 @llvm.vector.reduce.smin.nxv4i32( %v) @@ -1331,7 +1331,7 @@ define i64 @vreduce_umax_nxv1i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv1i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv1i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv1i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.umax.nxv1i64( %v) @@ -1346,7 +1346,7 @@ define i64 @vreduce_smax_nxv1i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv1i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv1i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv1i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.smax.nxv1i64( %v) @@ -1361,7 +1361,7 @@ define i64 @vreduce_umin_nxv1i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv1i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv1i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv1i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.umin.nxv1i64( %v) @@ -1376,7 +1376,7 @@ define i64 @vreduce_smin_nxv1i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv1i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv1i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv1i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.smin.nxv1i64( %v) @@ -1483,7 +1483,7 @@ define i64 @vreduce_umax_nxv2i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv2i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv2i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv2i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.umax.nxv2i64( %v) @@ -1498,7 +1498,7 @@ define i64 @vreduce_smax_nxv2i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv2i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv2i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv2i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.smax.nxv2i64( %v) @@ -1513,7 +1513,7 @@ define i64 @vreduce_umin_nxv2i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv2i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv2i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv2i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.umin.nxv2i64( %v) @@ -1528,7 +1528,7 @@ define i64 @vreduce_smin_nxv2i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv2i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv2i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv2i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.smin.nxv2i64( %v) @@ -1635,7 +1635,7 @@ define i64 @vreduce_umax_nxv4i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_umax_nxv4i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv4i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv4i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.umax.nxv4i64( %v) @@ -1650,7 +1650,7 @@ define i64 @vreduce_smax_nxv4i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_smax_nxv4i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv4i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv4i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.smax.nxv4i64( %v) @@ -1665,7 +1665,7 @@ define i64 @vreduce_umin_nxv4i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_umin_nxv4i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv4i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv4i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.umin.nxv4i64( %v) @@ -1680,7 +1680,7 @@ define i64 @vreduce_smin_nxv4i64( %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red ; ; SIZE-LABEL: 'vreduce_smin_nxv4i64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv4i64( %v) +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv4i64( %v) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red ; %red = call i64 @llvm.vector.reduce.smin.nxv4i64( %v) -- GitLab From 1679b27959bbe21b713725017b1cf36ac66dfadc Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Mon, 1 Apr 2024 08:46:57 +0200 Subject: [PATCH 0093/1690] [libc++] Refactor __tuple_like and __pair_like (#85206) The exposition-only type trait `pair-like` includes `ranges::subrange`, but in every single case excludes `ranges::subrange` from the list. This patch introduces two new traits `__tuple_like_no_subrange` and `__pair_like_no_subrange`, which exclude `ranges::subrange` from the possible matches. `__pair_like` is no longer required, and thus removed. `__tuple_like` is implemented as `__tuple_like_no_subrange` or a `ranges::subrange` specialization. --- libcxx/include/CMakeLists.txt | 2 +- libcxx/include/__algorithm/mismatch.h | 2 + .../__memory/uses_allocator_construction.h | 12 +--- .../__memory_resource/polymorphic_allocator.h | 1 + libcxx/include/__ranges/subrange.h | 7 +-- libcxx/include/__tuple/pair_like.h | 32 ---------- libcxx/include/__tuple/tuple_like.h | 32 +++------- .../include/__tuple/tuple_like_no_subrange.h | 61 +++++++++++++++++++ libcxx/include/__utility/pair.h | 27 +++----- libcxx/include/libcxx.imp | 2 +- libcxx/include/module.modulemap | 30 ++++----- libcxx/include/variant | 2 + .../tuple/__tuple_like.compile.pass.cpp | 1 + .../range.concept.compile.pass.cpp | 1 - 14 files changed, 108 insertions(+), 104 deletions(-) delete mode 100644 libcxx/include/__tuple/pair_like.h create mode 100644 libcxx/include/__tuple/tuple_like_no_subrange.h diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index 07b5e974eaf5..b935e45151e9 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -709,12 +709,12 @@ set(files __tree __tuple/find_index.h __tuple/make_tuple_types.h - __tuple/pair_like.h __tuple/sfinae_helpers.h __tuple/tuple_element.h __tuple/tuple_indices.h __tuple/tuple_like.h __tuple/tuple_like_ext.h + __tuple/tuple_like_no_subrange.h __tuple/tuple_size.h __tuple/tuple_types.h __type_traits/add_const.h diff --git a/libcxx/include/__algorithm/mismatch.h b/libcxx/include/__algorithm/mismatch.h index d933a84cada9..1cb83b01b2eb 100644 --- a/libcxx/include/__algorithm/mismatch.h +++ b/libcxx/include/__algorithm/mismatch.h @@ -18,10 +18,12 @@ #include <__type_traits/invoke.h> #include <__type_traits/is_constant_evaluated.h> #include <__type_traits/is_equality_comparable.h> +#include <__type_traits/is_integral.h> #include <__type_traits/operation_traits.h> #include <__utility/move.h> #include <__utility/pair.h> #include <__utility/unreachable.h> +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header diff --git a/libcxx/include/__memory/uses_allocator_construction.h b/libcxx/include/__memory/uses_allocator_construction.h index 71ae5bcd3233..9b7262bec5cf 100644 --- a/libcxx/include/__memory/uses_allocator_construction.h +++ b/libcxx/include/__memory/uses_allocator_construction.h @@ -12,7 +12,7 @@ #include <__config> #include <__memory/construct_at.h> #include <__memory/uses_allocator.h> -#include <__tuple/pair_like.h> +#include <__tuple/tuple_like_no_subrange.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_same.h> #include <__type_traits/remove_cv.h> @@ -128,11 +128,7 @@ __uses_allocator_construction_args(const _Alloc& __alloc, const pair<_Up, _Vp>&& std::forward_as_tuple(std::get<1>(std::move(__pair)))); } -template < class _Pair, - class _Alloc, - __pair_like _PairLike, - __enable_if_t<__is_cv_std_pair<_Pair> && !__is_specialization_of_subrange>::value, - int> = 0> +template , int> = 0> _LIBCPP_HIDE_FROM_ABI constexpr auto __uses_allocator_construction_args(const _Alloc& __alloc, _PairLike&& __p) noexcept { return std::__uses_allocator_construction_args<_Pair>( @@ -161,9 +157,7 @@ inline constexpr bool __convertible_to_const_pair_ref = # if _LIBCPP_STD_VER >= 23 template inline constexpr bool __uses_allocator_constraints = - __is_cv_std_pair<_Tp> && - (__is_specialization_of_subrange>::value || - (!__pair_like<_Up> && !__convertible_to_const_pair_ref<_Up>)); + __is_cv_std_pair<_Tp> && !__pair_like_no_subrange<_Up> && !__convertible_to_const_pair_ref<_Up>; # else template inline constexpr bool __uses_allocator_constraints = __is_cv_std_pair<_Tp> && !__convertible_to_const_pair_ref<_Up>; diff --git a/libcxx/include/__memory_resource/polymorphic_allocator.h b/libcxx/include/__memory_resource/polymorphic_allocator.h index cfd07bc84fe8..823c1503c22b 100644 --- a/libcxx/include/__memory_resource/polymorphic_allocator.h +++ b/libcxx/include/__memory_resource/polymorphic_allocator.h @@ -12,6 +12,7 @@ #include <__assert> #include <__availability> #include <__config> +#include <__fwd/pair.h> #include <__memory_resource/memory_resource.h> #include <__utility/exception_guard.h> #include diff --git a/libcxx/include/__ranges/subrange.h b/libcxx/include/__ranges/subrange.h index bb4411cf3554..051bc3113901 100644 --- a/libcxx/include/__ranges/subrange.h +++ b/libcxx/include/__ranges/subrange.h @@ -28,8 +28,8 @@ #include <__ranges/enable_borrowed_range.h> #include <__ranges/size.h> #include <__ranges/view_interface.h> -#include <__tuple/pair_like.h> #include <__tuple/tuple_element.h> +#include <__tuple/tuple_like_no_subrange.h> #include <__tuple/tuple_size.h> #include <__type_traits/conditional.h> #include <__type_traits/decay.h> @@ -64,7 +64,7 @@ concept __convertible_to_non_slicing = template concept __pair_like_convertible_from = - !range<_Pair> && __pair_like<_Pair> && constructible_from<_Pair, _Iter, _Sent> && + !range<_Pair> && __pair_like_no_subrange<_Pair> && constructible_from<_Pair, _Iter, _Sent> && __convertible_to_non_slicing<_Iter, tuple_element_t<0, _Pair>> && convertible_to<_Sent, tuple_element_t<1, _Pair>>; template _Pair> - requires __pair_like_convertible_from<_Pair, const _Iter&, const _Sent&> + template <__pair_like_convertible_from _Pair> _LIBCPP_HIDE_FROM_ABI constexpr operator _Pair() const { return _Pair(__begin_, __end_); } diff --git a/libcxx/include/__tuple/pair_like.h b/libcxx/include/__tuple/pair_like.h deleted file mode 100644 index 192682dc7eb5..000000000000 --- a/libcxx/include/__tuple/pair_like.h +++ /dev/null @@ -1,32 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef _LIBCPP___TUPLE_PAIR_LIKE_H -#define _LIBCPP___TUPLE_PAIR_LIKE_H - -#include <__config> -#include <__tuple/tuple_like.h> -#include <__tuple/tuple_size.h> -#include <__type_traits/remove_cvref.h> - -#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) -# pragma GCC system_header -#endif - -_LIBCPP_BEGIN_NAMESPACE_STD - -#if _LIBCPP_STD_VER >= 20 - -template -concept __pair_like = __tuple_like<_Tp> && tuple_size>::value == 2; - -#endif // _LIBCPP_STD_VER >= 20 - -_LIBCPP_END_NAMESPACE_STD - -#endif // _LIBCPP___TUPLE_PAIR_LIKE_H diff --git a/libcxx/include/__tuple/tuple_like.h b/libcxx/include/__tuple/tuple_like.h index 967e4a543c03..c080a3dcf1e2 100644 --- a/libcxx/include/__tuple/tuple_like.h +++ b/libcxx/include/__tuple/tuple_like.h @@ -10,12 +10,9 @@ #define _LIBCPP___TUPLE_TUPLE_LIKE_H #include <__config> -#include <__fwd/array.h> -#include <__fwd/complex.h> -#include <__fwd/pair.h> #include <__fwd/subrange.h> -#include <__fwd/tuple.h> -#include <__type_traits/integral_constant.h> +#include <__tuple/tuple_like_no_subrange.h> +#include <__tuple/tuple_size.h> #include <__type_traits/remove_cvref.h> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -27,29 +24,16 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 template -struct __tuple_like_impl : false_type {}; +inline constexpr bool __is_ranges_subrange_v = false; -template -struct __tuple_like_impl > : true_type {}; - -template -struct __tuple_like_impl > : true_type {}; - -template -struct __tuple_like_impl > : true_type {}; - -template -struct __tuple_like_impl > : true_type {}; - -# if _LIBCPP_STD_VER >= 26 +template +inline constexpr bool __is_ranges_subrange_v> = true; template -struct __tuple_like_impl> : true_type {}; - -# endif +concept __tuple_like = __tuple_like_no_subrange<_Tp> || __is_ranges_subrange_v>; -template -concept __tuple_like = __tuple_like_impl>::value; +// As of writing this comment every use of `pair-like` in the standard excludes `ranges::subrange`, so +// you most likely want to use `__pair_like_no_subrange` if you're looking for `pair-like`. #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__tuple/tuple_like_no_subrange.h b/libcxx/include/__tuple/tuple_like_no_subrange.h new file mode 100644 index 000000000000..274b0bf188e1 --- /dev/null +++ b/libcxx/include/__tuple/tuple_like_no_subrange.h @@ -0,0 +1,61 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H +#define _LIBCPP___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H + +#include <__config> +#include <__fwd/array.h> +#include <__fwd/complex.h> +#include <__fwd/pair.h> +#include <__fwd/tuple.h> +#include <__tuple/tuple_size.h> +#include <__type_traits/remove_cvref.h> +#include + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if _LIBCPP_STD_VER >= 20 + +template +inline constexpr bool __tuple_like_no_subrange_impl = false; + +template +inline constexpr bool __tuple_like_no_subrange_impl> = true; + +template +inline constexpr bool __tuple_like_no_subrange_impl> = true; + +template +inline constexpr bool __tuple_like_no_subrange_impl> = true; + +# if _LIBCPP_STD_VER >= 26 + +template +inline constexpr bool __tuple_like_no_subrange_impl> = true; + +# endif + +template +concept __tuple_like_no_subrange = __tuple_like_no_subrange_impl>; + +// This is equivalent to the exposition-only type trait `pair-like`, except that it is false for specializations of +// `ranges::subrange`. This is more useful than the pair-like concept in the standard because every use of `pair-like` +// excludes `ranges::subrange`. +template +concept __pair_like_no_subrange = __tuple_like_no_subrange<_Tp> && tuple_size>::value == 2; + +#endif // _LIBCPP_STD_VER >= 20 + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H diff --git a/libcxx/include/__utility/pair.h b/libcxx/include/__utility/pair.h index 0056806877e1..87b2f6f5e281 100644 --- a/libcxx/include/__utility/pair.h +++ b/libcxx/include/__utility/pair.h @@ -15,12 +15,11 @@ #include <__config> #include <__fwd/array.h> #include <__fwd/pair.h> -#include <__fwd/subrange.h> #include <__fwd/tuple.h> -#include <__tuple/pair_like.h> #include <__tuple/sfinae_helpers.h> #include <__tuple/tuple_element.h> #include <__tuple/tuple_indices.h> +#include <__tuple/tuple_like_no_subrange.h> #include <__tuple/tuple_size.h> #include <__type_traits/common_reference.h> #include <__type_traits/common_type.h> @@ -60,14 +59,6 @@ struct __non_trivially_copyable_base { __non_trivially_copyable_base(__non_trivially_copyable_base const&) _NOEXCEPT {} }; -#if _LIBCPP_STD_VER >= 23 -template -struct __is_specialization_of_subrange : false_type {}; - -template -struct __is_specialization_of_subrange> : true_type {}; -#endif - template struct _LIBCPP_TEMPLATE_VIS pair #if defined(_LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR) @@ -201,19 +192,19 @@ struct _LIBCPP_TEMPLATE_VIS pair # endif # if _LIBCPP_STD_VER >= 23 + // TODO: Remove this workaround in LLVM 20. The bug got fixed in Clang 18. // This is a workaround for http://llvm.org/PR60710. We should be able to remove it once Clang is fixed. template _LIBCPP_HIDE_FROM_ABI static constexpr bool __pair_like_explicit_wknd() { - if constexpr (__pair_like<_PairLike>) { + if constexpr (__pair_like_no_subrange<_PairLike>) { return !is_convertible_v(std::declval<_PairLike&&>())), first_type> || !is_convertible_v(std::declval<_PairLike&&>())), second_type>; } return false; } - template <__pair_like _PairLike> - requires(!__is_specialization_of_subrange>::value && - is_constructible_v(std::declval<_PairLike &&>()))> && + template <__pair_like_no_subrange _PairLike> + requires(is_constructible_v(std::declval<_PairLike &&>()))> && is_constructible_v(std::declval<_PairLike &&>()))>) _LIBCPP_HIDE_FROM_ABI constexpr explicit(__pair_like_explicit_wknd<_PairLike>()) pair(_PairLike&& __p) : first(std::get<0>(std::forward<_PairLike>(__p))), second(std::get<1>(std::forward<_PairLike>(__p))) {} @@ -306,8 +297,8 @@ struct _LIBCPP_TEMPLATE_VIS pair return *this; } - template <__pair_like _PairLike> - requires(__different_from<_PairLike, pair> && !__is_specialization_of_subrange>::value && + template <__pair_like_no_subrange _PairLike> + requires(__different_from<_PairLike, pair> && is_assignable_v(std::declval<_PairLike>()))> && is_assignable_v(std::declval<_PairLike>()))>) _LIBCPP_HIDE_FROM_ABI constexpr pair& operator=(_PairLike&& __p) { @@ -316,8 +307,8 @@ struct _LIBCPP_TEMPLATE_VIS pair return *this; } - template <__pair_like _PairLike> - requires(__different_from<_PairLike, pair> && !__is_specialization_of_subrange>::value && + template <__pair_like_no_subrange _PairLike> + requires(__different_from<_PairLike, pair> && is_assignable_v(std::declval<_PairLike>()))> && is_assignable_v(std::declval<_PairLike>()))>) _LIBCPP_HIDE_FROM_ABI constexpr pair const& operator=(_PairLike&& __p) const { diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp index fb446da5dc6f..ea0ba8c5471d 100644 --- a/libcxx/include/libcxx.imp +++ b/libcxx/include/libcxx.imp @@ -705,12 +705,12 @@ { include: [ "<__thread/timed_backoff_policy.h>", "private", "", "public" ] }, { include: [ "<__tuple/find_index.h>", "private", "", "public" ] }, { include: [ "<__tuple/make_tuple_types.h>", "private", "", "public" ] }, - { include: [ "<__tuple/pair_like.h>", "private", "", "public" ] }, { include: [ "<__tuple/sfinae_helpers.h>", "private", "", "public" ] }, { include: [ "<__tuple/tuple_element.h>", "private", "", "public" ] }, { include: [ "<__tuple/tuple_indices.h>", "private", "", "public" ] }, { include: [ "<__tuple/tuple_like.h>", "private", "", "public" ] }, { include: [ "<__tuple/tuple_like_ext.h>", "private", "", "public" ] }, + { include: [ "<__tuple/tuple_like_no_subrange.h>", "private", "", "public" ] }, { include: [ "<__tuple/tuple_size.h>", "private", "", "public" ] }, { include: [ "<__tuple/tuple_types.h>", "private", "", "public" ] }, { include: [ "<__type_traits/add_const.h>", "private", "", "public" ] }, diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap index 079c6234d410..85303d2fa62c 100644 --- a/libcxx/include/module.modulemap +++ b/libcxx/include/module.modulemap @@ -1810,20 +1810,22 @@ module std_private_thread_thread [system] { } module std_private_thread_timed_backoff_policy [system] { header "__thread/timed_backoff_policy.h" } -module std_private_tuple_find_index [system] { header "__tuple/find_index.h" } -module std_private_tuple_make_tuple_types [system] { header "__tuple/make_tuple_types.h" } -module std_private_tuple_pair_like [system] { - header "__tuple/pair_like.h" - export std_private_tuple_tuple_like -} -module std_private_tuple_sfinae_helpers [system] { header "__tuple/sfinae_helpers.h" } -module std_private_tuple_tuple_element [system] { header "__tuple/tuple_element.h" } -module std_private_tuple_tuple_fwd [system] { header "__fwd/tuple.h" } -module std_private_tuple_tuple_indices [system] { header "__tuple/tuple_indices.h" } -module std_private_tuple_tuple_like [system] { header "__tuple/tuple_like.h" } -module std_private_tuple_tuple_like_ext [system] { header "__tuple/tuple_like_ext.h" } -module std_private_tuple_tuple_size [system] { header "__tuple/tuple_size.h" } -module std_private_tuple_tuple_types [system] { header "__tuple/tuple_types.h" } +module std_private_tuple_find_index [system] { header "__tuple/find_index.h" } +module std_private_tuple_make_tuple_types [system] { header "__tuple/make_tuple_types.h" } +module std_private_tuple_tuple_like_no_subrange [system] { + header "__tuple/tuple_like_no_subrange.h" +} +module std_private_tuple_sfinae_helpers [system] { header "__tuple/sfinae_helpers.h" } +module std_private_tuple_tuple_element [system] { header "__tuple/tuple_element.h" } +module std_private_tuple_tuple_fwd [system] { header "__fwd/tuple.h" } +module std_private_tuple_tuple_indices [system] { header "__tuple/tuple_indices.h" } +module std_private_tuple_tuple_like [system] { + header "__tuple/tuple_like.h" + export * +} +module std_private_tuple_tuple_like_ext [system] { header "__tuple/tuple_like_ext.h" } +module std_private_tuple_tuple_size [system] { header "__tuple/tuple_size.h" } +module std_private_tuple_tuple_types [system] { header "__tuple/tuple_types.h" } module std_private_type_traits_add_const [system] { header "__type_traits/add_const.h" } module std_private_type_traits_add_cv [system] { header "__type_traits/add_cv.h" } diff --git a/libcxx/include/variant b/libcxx/include/variant index f5e99fc3239e..1b5e84e95479 100644 --- a/libcxx/include/variant +++ b/libcxx/include/variant @@ -235,11 +235,13 @@ namespace std { #include <__type_traits/is_destructible.h> #include <__type_traits/is_nothrow_assignable.h> #include <__type_traits/is_nothrow_constructible.h> +#include <__type_traits/is_reference.h> #include <__type_traits/is_trivially_assignable.h> #include <__type_traits/is_trivially_constructible.h> #include <__type_traits/is_trivially_destructible.h> #include <__type_traits/is_void.h> #include <__type_traits/remove_const.h> +#include <__type_traits/remove_cvref.h> #include <__type_traits/type_identity.h> #include <__type_traits/void_t.h> #include <__utility/declval.h> diff --git a/libcxx/test/libcxx/utilities/tuple/__tuple_like.compile.pass.cpp b/libcxx/test/libcxx/utilities/tuple/__tuple_like.compile.pass.cpp index b1d199865dec..2c54fb5a10e1 100644 --- a/libcxx/test/libcxx/utilities/tuple/__tuple_like.compile.pass.cpp +++ b/libcxx/test/libcxx/utilities/tuple/__tuple_like.compile.pass.cpp @@ -15,6 +15,7 @@ // template // concept tuple-like; // exposition only +#include <__tuple/tuple_like.h> #include #include #include diff --git a/libcxx/test/std/ranges/range.adaptors/range.elements/range.concept.compile.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.elements/range.concept.compile.pass.cpp index 7de72ecc0a90..a49e692d585d 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.elements/range.concept.compile.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.elements/range.concept.compile.pass.cpp @@ -52,7 +52,6 @@ static_assert(!std::ranges::input_range*>>, 0>); // !tuple-like -LIBCPP_STATIC_ASSERT(!std::__tuple_like); static_assert(!HasElementsView, 1>); // !(N < tuple_size_v) -- GitLab From d76a1233f7d7923d056a53cfa6f89735e9cda86e Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 31 Mar 2024 23:48:11 -0700 Subject: [PATCH 0094/1690] [CodeGen] Fix test after #86049 --- llvm/test/CodeGen/Generic/allow-check.ll | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llvm/test/CodeGen/Generic/allow-check.ll b/llvm/test/CodeGen/Generic/allow-check.ll index 493b7f8950b1..ba1f84e8398f 100644 --- a/llvm/test/CodeGen/Generic/allow-check.ll +++ b/llvm/test/CodeGen/Generic/allow-check.ll @@ -1,8 +1,9 @@ ; Avoid `!DL->isLittleEndian() && !CLI->enableBigEndian()` missmatch on PPC64BE. ; REQUIRES: host-byteorder-little-endian -; Does not support -global-isel=1. -; UNSUPPORTED: target=nvptx{{.*}} +; -global-isel=1 is unsupported. +; XFAIL: target=nvptx{{.*}} +; XFAIL: target=sparc{{.*}} ; RUN: llc < %s -O3 -global-isel=0 -fast-isel=0 ; RUN: llc < %s -O3 -global-isel=1 -fast-isel=0 -- GitLab From 8b135a7d1f59a5a7adccb162abf92d751209afe7 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Mon, 1 Apr 2024 00:37:35 -0700 Subject: [PATCH 0095/1690] [sanitizer][symbolizer] Add new dependency --- .../sanitizer_common/symbolizer/scripts/build_symbolizer.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh index 7f4082a6cd19..005bd6d584c5 100755 --- a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh +++ b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh @@ -144,7 +144,7 @@ if [[ ! -f ${LLVM_BUILD}/build.ninja ]]; then $LLVM_SRC fi cd ${LLVM_BUILD} -ninja LLVMSymbolize LLVMObject LLVMBinaryFormat LLVMDebugInfoDWARF LLVMSupport LLVMDebugInfoPDB LLVMDebuginfod LLVMMC LLVMDemangle LLVMTextAPI LLVMTargetParser +ninja LLVMSymbolize LLVMObject LLVMBinaryFormat LLVMDebugInfoDWARF LLVMSupport LLVMDebugInfoPDB LLVMDebuginfod LLVMMC LLVMDemangle LLVMTextAPI LLVMTargetParser LLVMCore cd ${BUILD_DIR} rm -rf ${SYMBOLIZER_BUILD} @@ -182,6 +182,7 @@ $LINK $LIBCXX_ARCHIVE_DIR/libc++.a \ $LLVM_BUILD/lib/libLLVMMC.a \ $LLVM_BUILD/lib/libLLVMTextAPI.a \ $LLVM_BUILD/lib/libLLVMTargetParser.a \ + $LLVM_BUILD/lib/libLLVMCore.a \ $ZLIB_BUILD/libz.a \ symbolizer.a \ -ignore-non-bitcode -o all.bc -- GitLab From c0cabfbdaf547b1152065be5419bc48e0b83b761 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Mon, 1 Apr 2024 00:42:18 -0700 Subject: [PATCH 0096/1690] [InstCombiner] Remove trivially dead `llvm.allow.{runtime,ubsan}.check()` (#84851) Intrinsic declared to have sideeffects, but it's done only to prevent moving it. Removing unused ones is OK. Exacted from #84850 for easier review. --- llvm/lib/Transforms/Utils/Local.cpp | 6 +++ .../Transforms/InstCombine/allow-checks.ll | 44 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 llvm/test/Transforms/InstCombine/allow-checks.ll diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index 3d3b97eca92b..380bac9c6180 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -475,6 +475,12 @@ bool llvm::wouldInstructionBeTriviallyDead(const Instruction *I, II->getIntrinsicID() == Intrinsic::launder_invariant_group) return true; + // Intrinsics declare sideeffects to prevent them from moving, but they are + // nops without users. + if (II->getIntrinsicID() == Intrinsic::allow_runtime_check || + II->getIntrinsicID() == Intrinsic::allow_ubsan_check) + return true; + if (II->isLifetimeStartOrEnd()) { auto *Arg = II->getArgOperand(1); // Lifetime intrinsics are dead when their right-hand is undef. diff --git a/llvm/test/Transforms/InstCombine/allow-checks.ll b/llvm/test/Transforms/InstCombine/allow-checks.ll new file mode 100644 index 000000000000..873b7c3014a8 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/allow-checks.ll @@ -0,0 +1,44 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -passes=instcombine -S | FileCheck %s --implicit-check-not="call i1 @llvm.allow" + +define i1 @test_runtime() { +; CHECK-LABEL: @test_runtime( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[HOT:%.*]] = call i1 @llvm.allow.runtime.check(metadata !"test") +; CHECK-NEXT: ret i1 [[HOT]] +; +entry: + %allow = call i1 @llvm.allow.runtime.check(metadata !"test") + ret i1 %allow +} + +define void @test_runtime_void() { +; CHECK-LABEL: @test_runtime_void( +; CHECK-NEXT: entry: +; CHECK-NEXT: ret void +; +entry: + %allow = call i1 @llvm.allow.runtime.check(metadata !"test") + ret void +} + +define i1 @test_ubsan() { +; CHECK-LABEL: @test_ubsan( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[HOT:%.*]] = call i1 @llvm.allow.ubsan.check(i8 11) +; CHECK-NEXT: ret i1 [[HOT]] +; +entry: + %allow = call i1 @llvm.allow.ubsan.check(i8 11) + ret i1 %allow +} + +define void @test_ubsan_void() { +; CHECK-LABEL: @test_ubsan_void( +; CHECK-NEXT: entry: +; CHECK-NEXT: ret void +; +entry: + %allow = call i1 @llvm.allow.ubsan.check(i8 11) + ret void +} -- GitLab From cbb27bef3e53c75d7272a6392cee56c763368111 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Mon, 1 Apr 2024 00:44:17 -0700 Subject: [PATCH 0097/1690] [CodeGen] Fix test after #86049 --- llvm/test/CodeGen/Generic/allow-check.ll | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/test/CodeGen/Generic/allow-check.ll b/llvm/test/CodeGen/Generic/allow-check.ll index ba1f84e8398f..43dab6802aca 100644 --- a/llvm/test/CodeGen/Generic/allow-check.ll +++ b/llvm/test/CodeGen/Generic/allow-check.ll @@ -4,6 +4,7 @@ ; -global-isel=1 is unsupported. ; XFAIL: target=nvptx{{.*}} ; XFAIL: target=sparc{{.*}} +; XFAIL: target=hexagon-{{.*}} ; RUN: llc < %s -O3 -global-isel=0 -fast-isel=0 ; RUN: llc < %s -O3 -global-isel=1 -fast-isel=0 -- GitLab From 10a57f3aff34be6ab43106dc1e45ace3f6da881c Mon Sep 17 00:00:00 2001 From: Prashant Kumar Date: Mon, 1 Apr 2024 13:18:27 +0530 Subject: [PATCH 0098/1690] [mlir][math] Expand powfI operation for constant power operand. (#87081) -- Convert `math.fpowi` to a series of `arith.mulf` operations. -- If the power is negative, we divide the result by 1. --- .../mlir/Dialect/Math/Transforms/Passes.h | 1 + .../Math/Transforms/ExpandPatterns.cpp | 87 +++++++++++++++- mlir/test/Dialect/Math/expand-math.mlir | 99 +++++++++++++++++++ mlir/test/lib/Dialect/Math/TestExpandMath.cpp | 1 + 4 files changed, 183 insertions(+), 5 deletions(-) diff --git a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h index 11b2c7a7afa2..e2c513047c77 100644 --- a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h +++ b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h @@ -36,6 +36,7 @@ void populateExpandFloorFPattern(RewritePatternSet &patterns); void populateExpandCeilFPattern(RewritePatternSet &patterns); void populateExpandExp2FPattern(RewritePatternSet &patterns); void populateExpandPowFPattern(RewritePatternSet &patterns); +void populateExpandFPowIPattern(RewritePatternSet &patterns); void populateExpandRoundFPattern(RewritePatternSet &patterns); void populateExpandRoundEvenPattern(RewritePatternSet &patterns); void populateMathAlgebraicSimplificationPatterns(RewritePatternSet &patterns); diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp index e1ab9c905447..0b8546251350 100644 --- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp +++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp @@ -1,4 +1,4 @@ -//===- ExpandTanh.cpp - Code to perform expanding tanh op -----------------===// +//===- ExpandPatterns.cpp - Code to expand various math operations. -------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// This file implements expansion of tanh op. +// This file implements expansion of various math operations. // //===----------------------------------------------------------------------===// @@ -23,9 +23,14 @@ using namespace mlir; /// Create a float constant. -static Value createFloatConst(Location loc, Type type, double value, +static Value createFloatConst(Location loc, Type type, APFloat value, OpBuilder &b) { - auto attr = b.getFloatAttr(getElementTypeOrSelf(type), value); + bool losesInfo = false; + auto eltType = getElementTypeOrSelf(type); + // Convert double to the given `FloatType` with round-to-nearest-ties-to-even. + value.convert(cast(eltType).getFloatSemantics(), + APFloat::rmNearestTiesToEven, &losesInfo); + auto attr = b.getFloatAttr(eltType, value); if (auto shapedTy = dyn_cast(type)) { return b.create(loc, DenseElementsAttr::get(shapedTy, attr)); @@ -34,7 +39,12 @@ static Value createFloatConst(Location loc, Type type, double value, return b.create(loc, attr); } -/// Create a float constant. +static Value createFloatConst(Location loc, Type type, double value, + OpBuilder &b) { + return createFloatConst(loc, type, APFloat(value), b); +} + +/// Create an integer constant. static Value createIntConst(Location loc, Type type, int64_t value, OpBuilder &b) { auto attr = b.getIntegerAttr(getElementTypeOrSelf(type), value); @@ -202,6 +212,69 @@ static LogicalResult convertCeilOp(math::CeilOp op, PatternRewriter &rewriter) { rewriter.replaceOp(op, ret); return success(); } + +// Convert `math.fpowi` to a series of `arith.mulf` operations. +// If the power is negative, we divide one by the result. +// If both the base and power are zero, the result is 1. +static LogicalResult convertFPowICstOp(math::FPowIOp op, + PatternRewriter &rewriter) { + ImplicitLocOpBuilder b(op->getLoc(), rewriter); + Value base = op.getOperand(0); + Value power = op.getOperand(1); + Type baseType = base.getType(); + + Attribute cstAttr; + if (!matchPattern(power, m_Constant(&cstAttr))) + return failure(); + + APInt value; + if (!matchPattern(cstAttr, m_ConstantInt(&value))) + return failure(); + + int64_t powerInt = value.getSExtValue(); + bool isNegative = powerInt < 0; + int64_t absPower = std::abs(powerInt); + Value one = createFloatConst(op->getLoc(), baseType, 1.00, rewriter); + Value res = createFloatConst(op->getLoc(), baseType, 1.00, rewriter); + + while (absPower > 0) { + if (absPower & 1) + res = b.create(baseType, base, res); + absPower >>= 1; + base = b.create(baseType, base, base); + } + + // Make sure not to introduce UB in case of negative power. + if (isNegative) { + auto &sem = dyn_cast(getElementTypeOrSelf(baseType)) + .getFloatSemantics(); + Value zero = + createFloatConst(op->getLoc(), baseType, + APFloat::getZero(sem, /*Negative=*/false), rewriter); + Value negZero = + createFloatConst(op->getLoc(), baseType, + APFloat::getZero(sem, /*Negative=*/true), rewriter); + Value posInfinity = + createFloatConst(op->getLoc(), baseType, + APFloat::getInf(sem, /*Negative=*/false), rewriter); + Value negInfinity = + createFloatConst(op->getLoc(), baseType, + APFloat::getInf(sem, /*Negative=*/true), rewriter); + Value zeroEqCheck = + b.create(arith::CmpFPredicate::OEQ, res, zero); + Value negZeroEqCheck = + b.create(arith::CmpFPredicate::OEQ, res, negZero); + res = b.create(baseType, one, res); + res = + b.create(op->getLoc(), zeroEqCheck, posInfinity, res); + res = b.create(op->getLoc(), negZeroEqCheck, negInfinity, + res); + } + + rewriter.replaceOp(op, res); + return success(); +} + // Converts Powf(float a, float b) (meaning a^b) to exp^(b * ln(a)) static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) { ImplicitLocOpBuilder b(op->getLoc(), rewriter); @@ -517,6 +590,10 @@ void mlir::populateExpandPowFPattern(RewritePatternSet &patterns) { patterns.add(convertPowfOp); } +void mlir::populateExpandFPowIPattern(RewritePatternSet &patterns) { + patterns.add(convertFPowICstOp); +} + void mlir::populateExpandRoundFPattern(RewritePatternSet &patterns) { patterns.add(convertRoundOp); } diff --git a/mlir/test/Dialect/Math/expand-math.mlir b/mlir/test/Dialect/Math/expand-math.mlir index 6326d3a71874..bfcff27bd64e 100644 --- a/mlir/test/Dialect/Math/expand-math.mlir +++ b/mlir/test/Dialect/Math/expand-math.mlir @@ -511,3 +511,102 @@ func.func @roundeven16(%arg: f16) -> f16 { // CHECK: %[[COPYSIGN:.*]] = math.copysign %[[RESULT]], %[[VAL_0]] : f16 // CHECK: return %[[COPYSIGN]] : f16 + +// ----- + +// CHECK-LABEL: func.func @math_fpowi_neg_odd_power +func.func @math_fpowi_neg_odd_power(%0 : tensor<8xf32>) -> tensor<8xf32> { + %1 = arith.constant dense<-3> : tensor<8xi64> + %2 = math.fpowi %0, %1 : tensor<8xf32>, tensor<8xi64> + return %2 : tensor<8xf32> +} +// CHECK-SAME: (%[[ARG0:.*]]: tensor<8xf32>) -> tensor<8xf32> { +// CHECK-DAG: %[[CST1:.*]] = arith.constant dense<1.000000e+00> : tensor<8xf32> +// CHECK-DAG: %[[CST0:.*]] = arith.constant dense<0.000000e+00> : tensor<8xf32> +// CHECK-DAG: %[[CSTNEG0:.*]] = arith.constant dense<-0.000000e+00> : tensor<8xf32> +// CHECK-DAG: %[[CSTINF:.*]] = arith.constant dense<0x7F800000> : tensor<8xf32> +// CHECK-DAG: %[[CSTNEGINF:.*]] = arith.constant dense<0xFF800000> : tensor<8xf32> +// CHECK: %[[SQ:.*]] = arith.mulf %[[ARG0]], %[[ARG0]] : tensor<8xf32> +// CHECK: %[[CUBE:.*]] = arith.mulf %[[SQ]], %[[ARG0]] : tensor<8xf32> +// CHECK: %[[CMP0:.*]] = arith.cmpf oeq, %[[CUBE]], %[[CST0]] : tensor<8xf32> +// CHECK: %[[CMPNEG0:.*]] = arith.cmpf oeq, %[[CUBE]], %[[CSTNEG0]] : tensor<8xf32> +// CHECK: %[[INV:.*]] = arith.divf %[[CST1]], %[[CUBE]] : tensor<8xf32> +// CHECK: %[[UB1:.*]] = arith.select %[[CMP0]], %[[CSTINF]], %[[INV]] : tensor<8xi1>, tensor<8xf32> +// CHECK: %[[UB2:.*]] = arith.select %[[CMPNEG0]], %[[CSTNEGINF]], %[[UB1]] : tensor<8xi1>, tensor<8xf32> +// CHECK: return %[[UB2]] : tensor<8xf32> + +// ----- + +// CHECK-LABEL: func.func @math_fpowi_neg_even_power +func.func @math_fpowi_neg_even_power(%0 : tensor<8xf32>) -> tensor<8xf32> { + %1 = arith.constant dense<-4> : tensor<8xi64> + %2 = math.fpowi %0, %1 : tensor<8xf32>, tensor<8xi64> + return %2 : tensor<8xf32> +} +// CHECK-SAME: (%[[ARG0:.*]]: tensor<8xf32>) -> tensor<8xf32> { +// CHECK-DAG: %[[CST1:.*]] = arith.constant dense<1.000000e+00> : tensor<8xf32> +// CHECK-DAG: %[[CST0:.*]] = arith.constant dense<0.000000e+00> : tensor<8xf32> +// CHECK-DAG: %[[CSTNEG0:.*]] = arith.constant dense<-0.000000e+00> : tensor<8xf32> +// CHECK-DAG: %[[CSTINF:.*]] = arith.constant dense<0x7F800000> : tensor<8xf32> +// CHECK-DAG: %[[CSTNEGINF:.*]] = arith.constant dense<0xFF800000> : tensor<8xf32> +// CHECK: %[[SQ:.*]] = arith.mulf %[[ARG0]], %[[ARG0]] : tensor<8xf32> +// CHECK: %[[PW4:.*]] = arith.mulf %[[SQ]], %[[SQ]] : tensor<8xf32> +// CHECK: %[[CMP0:.*]] = arith.cmpf oeq, %[[PW4]], %[[CST0]] : tensor<8xf32> +// CHECK: %[[CMPNEG0:.*]] = arith.cmpf oeq, %[[PW4]], %[[CSTNEG0]] : tensor<8xf32> +// CHECK: %[[INV:.*]] = arith.divf %[[CST1]], %[[PW4]] : tensor<8xf32> +// CHECK: %[[UB1:.*]] = arith.select %[[CMP0]], %[[CSTINF]], %[[INV]] : tensor<8xi1>, tensor<8xf32> +// CHECK: %[[UB2:.*]] = arith.select %[[CMPNEG0]], %[[CSTNEGINF]], %[[UB1]] : tensor<8xi1>, tensor<8xf32> +// CHECK: return %[[UB2]] : tensor<8xf32> + +// ----- + +// CHECK-LABEL: func.func @math_fpowi_pos_odd_power +func.func @math_fpowi_pos_odd_power(%0 : tensor<8xf32>) -> tensor<8xf32> { + %1 = arith.constant dense<5> : tensor<8xi64> + %2 = math.fpowi %0, %1 : tensor<8xf32>, tensor<8xi64> + return %2 : tensor<8xf32> +} +// CHECK-SAME: (%[[ARG0:.*]]: tensor<8xf32>) -> tensor<8xf32> { +// CHECK: %[[SQ:.*]] = arith.mulf %[[ARG0]], %[[ARG0]] : tensor<8xf32> +// CHECK: %[[PW4:.*]] = arith.mulf %[[SQ]], %[[SQ]] : tensor<8xf32> +// CHECK: %[[PW5:.*]] = arith.mulf %[[PW4]], %[[ARG0]] : tensor<8xf32> +// CHECK: return %[[PW5]] : tensor<8xf32> + +// ----- + +// CHECK-LABEL: func.func @math_fpowi_pos_even_power +func.func @math_fpowi_pos_even_power(%0 : tensor<8xf32>) -> tensor<8xf32> { + %1 = arith.constant dense<4> : tensor<8xi64> + %2 = math.fpowi %0, %1 : tensor<8xf32>, tensor<8xi64> + return %2 : tensor<8xf32> +} +// CHECK-SAME: (%[[ARG0:.*]]: tensor<8xf32>) -> tensor<8xf32> { +// CHECK: %[[SQ:.*]] = arith.mulf %[[ARG0]], %[[ARG0]] : tensor<8xf32> +// CHECK: %[[PW4:.*]] = arith.mulf %[[SQ]], %[[SQ]] : tensor<8xf32> +// CHECK: return %[[PW4]] : tensor<8xf32> + +// ----- + +// CHECK-LABEL: func.func @math_fpowi_even_scalar +func.func @math_fpowi_even_scalar(%0 : f32) -> f32 { + %pow = arith.constant 2 : i64 + %2 = math.fpowi %0, %pow : f32, i64 + return %2 : f32 +} +// CHECK-SAME: (%[[ARG0:.*]]: f32) -> f32 { +// CHECK: %[[SQ:.*]] = arith.mulf %[[ARG0]], %[[ARG0]] : f32 +// CHECK: return %[[SQ]] : f32 + +// ----- + +// CHECK-LABEL: func.func @math_fpowi_scalar_zero +func.func @math_fpowi_scalar_zero(%0 : f32) -> f32 { + %pow = arith.constant 0 : i64 + %2 = math.fpowi %0, %pow : f32, i64 + return %2 : f32 +} +// CHECK-SAME: (%[[ARG0:.*]]: f32) -> f32 { +// CHECK: %[[RET:.*]] = arith.constant 1.000000e+00 : f32 +// CHECK: return %[[RET]] : f32 + +// ----- diff --git a/mlir/test/lib/Dialect/Math/TestExpandMath.cpp b/mlir/test/lib/Dialect/Math/TestExpandMath.cpp index 7ce8b5a7cfe9..97600ad1ebe7 100644 --- a/mlir/test/lib/Dialect/Math/TestExpandMath.cpp +++ b/mlir/test/lib/Dialect/Math/TestExpandMath.cpp @@ -46,6 +46,7 @@ void TestExpandMathPass::runOnOperation() { populateExpandFloorFPattern(patterns); populateExpandCeilFPattern(patterns); populateExpandPowFPattern(patterns); + populateExpandFPowIPattern(patterns); populateExpandRoundFPattern(patterns); populateExpandRoundEvenPattern(patterns); (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns)); -- GitLab From 3365d62179011aad6da3e4cbcb31044eec3462a2 Mon Sep 17 00:00:00 2001 From: Congcong Cai Date: Mon, 1 Apr 2024 15:53:57 +0800 Subject: [PATCH 0099/1690] [clang-tidy] add new check readability-enum-initial-value (#86129) Fixes: #85243. --- .../clang-tidy/readability/CMakeLists.txt | 1 + .../readability/EnumInitialValueCheck.cpp | 200 ++++++++++++++++++ .../readability/EnumInitialValueCheck.h | 38 ++++ .../readability/ReadabilityTidyModule.cpp | 3 + clang-tools-extra/docs/ReleaseNotes.rst | 6 + .../docs/clang-tidy/checks/list.rst | 1 + .../checks/readability/enum-initial-value.rst | 75 +++++++ .../checkers/readability/enum-initial-value.c | 80 +++++++ .../readability/enum-initial-value.cpp | 27 +++ 9 files changed, 431 insertions(+) create mode 100644 clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp create mode 100644 clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.h create mode 100644 clang-tools-extra/docs/clang-tidy/checks/readability/enum-initial-value.rst create mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c create mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.cpp diff --git a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt index 5728c9970fb6..dd772d692025 100644 --- a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt +++ b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt @@ -17,6 +17,7 @@ add_clang_library(clangTidyReadabilityModule DeleteNullPointerCheck.cpp DuplicateIncludeCheck.cpp ElseAfterReturnCheck.cpp + EnumInitialValueCheck.cpp FunctionCognitiveComplexityCheck.cpp FunctionSizeCheck.cpp IdentifierLengthCheck.cpp diff --git a/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp b/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp new file mode 100644 index 000000000000..8f2841c32259 --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp @@ -0,0 +1,200 @@ +//===--- EnumInitialValueCheck.cpp - clang-tidy ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "EnumInitialValueCheck.h" +#include "../utils/LexerUtils.h" +#include "clang/AST/Decl.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/ASTMatchers/ASTMatchers.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/SourceLocation.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +static bool isNoneEnumeratorsInitialized(const EnumDecl &Node) { + return llvm::all_of(Node.enumerators(), [](const EnumConstantDecl *ECD) { + return ECD->getInitExpr() == nullptr; + }); +} + +static bool isOnlyFirstEnumeratorInitialized(const EnumDecl &Node) { + bool IsFirst = true; + for (const EnumConstantDecl *ECD : Node.enumerators()) { + if ((IsFirst && ECD->getInitExpr() == nullptr) || + (!IsFirst && ECD->getInitExpr() != nullptr)) + return false; + IsFirst = false; + } + return !IsFirst; +} + +static bool areAllEnumeratorsInitialized(const EnumDecl &Node) { + return llvm::all_of(Node.enumerators(), [](const EnumConstantDecl *ECD) { + return ECD->getInitExpr() != nullptr; + }); +} + +/// Check if \p Enumerator is initialized with a (potentially negated) \c +/// IntegerLiteral. +static bool isInitializedByLiteral(const EnumConstantDecl *Enumerator) { + const Expr *const Init = Enumerator->getInitExpr(); + if (!Init) + return false; + return Init->isIntegerConstantExpr(Enumerator->getASTContext()); +} + +static void cleanInitialValue(DiagnosticBuilder &Diag, + const EnumConstantDecl *ECD, + const SourceManager &SM, + const LangOptions &LangOpts) { + const SourceRange InitExprRange = ECD->getInitExpr()->getSourceRange(); + if (InitExprRange.isInvalid() || InitExprRange.getBegin().isMacroID() || + InitExprRange.getEnd().isMacroID()) + return; + std::optional EqualToken = utils::lexer::findNextTokenSkippingComments( + ECD->getLocation(), SM, LangOpts); + if (!EqualToken.has_value() || + EqualToken.value().getKind() != tok::TokenKind::equal) + return; + const SourceLocation EqualLoc{EqualToken->getLocation()}; + if (EqualLoc.isInvalid() || EqualLoc.isMacroID()) + return; + Diag << FixItHint::CreateRemoval(EqualLoc) + << FixItHint::CreateRemoval(InitExprRange); + return; +} + +namespace { + +AST_MATCHER(EnumDecl, isMacro) { + SourceLocation Loc = Node.getBeginLoc(); + return Loc.isMacroID(); +} + +AST_MATCHER(EnumDecl, hasConsistentInitialValues) { + return isNoneEnumeratorsInitialized(Node) || + isOnlyFirstEnumeratorInitialized(Node) || + areAllEnumeratorsInitialized(Node); +} + +AST_MATCHER(EnumDecl, hasZeroInitialValueForFirstEnumerator) { + const EnumDecl::enumerator_range Enumerators = Node.enumerators(); + if (Enumerators.empty()) + return false; + const EnumConstantDecl *ECD = *Enumerators.begin(); + return isOnlyFirstEnumeratorInitialized(Node) && + isInitializedByLiteral(ECD) && ECD->getInitVal().isZero(); +} + +/// Excludes bitfields because enumerators initialized with the result of a +/// bitwise operator on enumeration values or any other expr that is not a +/// potentially negative integer literal. +/// Enumerations where it is not directly clear if they are used with +/// bitmask, evident when enumerators are only initialized with (potentially +/// negative) integer literals, are ignored. This is also the case when all +/// enumerators are powers of two (e.g., 0, 1, 2). +AST_MATCHER(EnumDecl, hasSequentialInitialValues) { + const EnumDecl::enumerator_range Enumerators = Node.enumerators(); + if (Enumerators.empty()) + return false; + const EnumConstantDecl *const FirstEnumerator = *Node.enumerator_begin(); + llvm::APSInt PrevValue = FirstEnumerator->getInitVal(); + if (!isInitializedByLiteral(FirstEnumerator)) + return false; + bool AllEnumeratorsArePowersOfTwo = true; + for (const EnumConstantDecl *Enumerator : llvm::drop_begin(Enumerators)) { + const llvm::APSInt NewValue = Enumerator->getInitVal(); + if (NewValue != ++PrevValue) + return false; + if (!isInitializedByLiteral(Enumerator)) + return false; + PrevValue = NewValue; + AllEnumeratorsArePowersOfTwo &= NewValue.isPowerOf2(); + } + return !AllEnumeratorsArePowersOfTwo; +} + +} // namespace + +EnumInitialValueCheck::EnumInitialValueCheck(StringRef Name, + ClangTidyContext *Context) + : ClangTidyCheck(Name, Context), + AllowExplicitZeroFirstInitialValue( + Options.get("AllowExplicitZeroFirstInitialValue", true)), + AllowExplicitSequentialInitialValues( + Options.get("AllowExplicitSequentialInitialValues", true)) {} + +void EnumInitialValueCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { + Options.store(Opts, "AllowExplicitZeroFirstInitialValue", + AllowExplicitZeroFirstInitialValue); + Options.store(Opts, "AllowExplicitSequentialInitialValues", + AllowExplicitSequentialInitialValues); +} + +void EnumInitialValueCheck::registerMatchers(MatchFinder *Finder) { + Finder->addMatcher( + enumDecl(unless(isMacro()), unless(hasConsistentInitialValues())) + .bind("inconsistent"), + this); + if (!AllowExplicitZeroFirstInitialValue) + Finder->addMatcher( + enumDecl(hasZeroInitialValueForFirstEnumerator()).bind("zero_first"), + this); + if (!AllowExplicitSequentialInitialValues) + Finder->addMatcher(enumDecl(unless(isMacro()), hasSequentialInitialValues()) + .bind("sequential"), + this); +} + +void EnumInitialValueCheck::check(const MatchFinder::MatchResult &Result) { + if (const auto *Enum = Result.Nodes.getNodeAs("inconsistent")) { + DiagnosticBuilder Diag = + diag(Enum->getBeginLoc(), + "inital values in enum %0 are not consistent, consider explicit " + "initialization of all, none or only the first enumerator") + << Enum; + for (const EnumConstantDecl *ECD : Enum->enumerators()) + if (ECD->getInitExpr() == nullptr) { + const SourceLocation EndLoc = Lexer::getLocForEndOfToken( + ECD->getLocation(), 0, *Result.SourceManager, getLangOpts()); + if (EndLoc.isMacroID()) + continue; + llvm::SmallString<8> Str{" = "}; + ECD->getInitVal().toString(Str); + Diag << FixItHint::CreateInsertion(EndLoc, Str); + } + return; + } + + if (const auto *Enum = Result.Nodes.getNodeAs("zero_first")) { + const EnumConstantDecl *ECD = *Enum->enumerator_begin(); + const SourceLocation Loc = ECD->getLocation(); + if (Loc.isInvalid() || Loc.isMacroID()) + return; + DiagnosticBuilder Diag = diag(Loc, "zero initial value for the first " + "enumerator in %0 can be disregarded") + << Enum; + cleanInitialValue(Diag, ECD, *Result.SourceManager, getLangOpts()); + return; + } + if (const auto *Enum = Result.Nodes.getNodeAs("sequential")) { + DiagnosticBuilder Diag = + diag(Enum->getBeginLoc(), + "sequential initial value in %0 can be ignored") + << Enum; + for (const EnumConstantDecl *ECD : llvm::drop_begin(Enum->enumerators())) + cleanInitialValue(Diag, ECD, *Result.SourceManager, getLangOpts()); + return; + } +} + +} // namespace clang::tidy::readability diff --git a/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.h b/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.h new file mode 100644 index 000000000000..66087e4ee170 --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.h @@ -0,0 +1,38 @@ +//===--- EnumInitialValueCheck.h - clang-tidy -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_ENUMINITIALVALUECHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_ENUMINITIALVALUECHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Enforces consistent style for enumerators' initialization, covering three +/// styles: none, first only, or all initialized explicitly. +/// +/// For the user-facing documentation see: +/// http://clang.llvm.org/extra/clang-tidy/checks/readability/enum-initial-value.html +class EnumInitialValueCheck : public ClangTidyCheck { +public: + EnumInitialValueCheck(StringRef Name, ClangTidyContext *Context); + void storeOptions(ClangTidyOptions::OptionMap &Opts) override; + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; + std::optional getCheckTraversalKind() const override { + return TK_IgnoreUnlessSpelledInSource; + } + +private: + const bool AllowExplicitZeroFirstInitialValue; + const bool AllowExplicitSequentialInitialValues; +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_ENUMINITIALVALUECHECK_H diff --git a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp index bca2c425111f..376b84683df7 100644 --- a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp +++ b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp @@ -22,6 +22,7 @@ #include "DeleteNullPointerCheck.h" #include "DuplicateIncludeCheck.h" #include "ElseAfterReturnCheck.h" +#include "EnumInitialValueCheck.h" #include "FunctionCognitiveComplexityCheck.h" #include "FunctionSizeCheck.h" #include "IdentifierLengthCheck.h" @@ -92,6 +93,8 @@ public: "readability-duplicate-include"); CheckFactories.registerCheck( "readability-else-after-return"); + CheckFactories.registerCheck( + "readability-enum-initial-value"); CheckFactories.registerCheck( "readability-function-cognitive-complexity"); CheckFactories.registerCheck( diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 78b09d23d442..309b844615a1 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -123,6 +123,12 @@ New checks Finds initializer lists for aggregate types that could be written as designated initializers instead. +- New :doc:`readability-enum-initial-value + ` check. + + Enforces consistent style for enumerators' initialization, covering three + styles: none, first only, or all initialized explicitly. + - New :doc:`readability-use-std-min-max ` check. diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst index 79e81dd174e4..188a42bfddd3 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/list.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst @@ -352,6 +352,7 @@ Clang-Tidy Checks :doc:`readability-delete-null-pointer `, "Yes" :doc:`readability-duplicate-include `, "Yes" :doc:`readability-else-after-return `, "Yes" + :doc:`readability-enum-initial-value `, "Yes" :doc:`readability-function-cognitive-complexity `, :doc:`readability-function-size `, :doc:`readability-identifier-length `, diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/enum-initial-value.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/enum-initial-value.rst new file mode 100644 index 000000000000..660efc1eaff3 --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/readability/enum-initial-value.rst @@ -0,0 +1,75 @@ +.. title:: clang-tidy - readability-enum-initial-value + +readability-enum-initial-value +============================== + +Enforces consistent style for enumerators' initialization, covering three +styles: none, first only, or all initialized explicitly. + +When adding new enumerations, inconsistent initial value will cause potential +enumeration value conflicts. + +In an enumeration, the following three cases are accepted. +1. none of enumerators are explicit initialized. +2. the first enumerator is explicit initialized. +3. all of enumerators are explicit initialized. + +.. code-block:: c++ + + // valid, none of enumerators are initialized. + enum A { + e0, + e1, + e2, + }; + + // valid, the first enumerator is initialized. + enum A { + e0 = 0, + e1, + e2, + }; + + // valid, all of enumerators are initialized. + enum A { + e0 = 0, + e1 = 1, + e2 = 2, + }; + + // invalid, e1 is not explicit initialized. + enum A { + e0 = 0, + e1, + e2 = 2, + }; + +Options +------- + +.. option:: AllowExplicitZeroFirstInitialValue + + If set to `false`, the first enumerator must not be explicitly initialized. + See examples below. Default is `true`. + + .. code-block:: c++ + + enum A { + e0 = 0, // not allowed if AllowExplicitZeroFirstInitialValue is false + e1, + e2, + }; + + +.. option:: AllowExplicitSequentialInitialValues + + If set to `false`, sequential initializations are not allowed. + See examples below. Default is `true`. + + .. code-block:: c++ + + enum A { + e0 = 1, // not allowed if AllowExplicitSequentialInitialValues is false + e1 = 2, + e2 = 3, + }; diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c b/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c new file mode 100644 index 000000000000..c66288cbe3e9 --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c @@ -0,0 +1,80 @@ +// RUN: %check_clang_tidy %s readability-enum-initial-value %t +// RUN: %check_clang_tidy -check-suffix=ENABLE %s readability-enum-initial-value %t -- \ +// RUN: -config='{CheckOptions: { \ +// RUN: readability-enum-initial-value.AllowExplicitZeroFirstInitialValue: false, \ +// RUN: readability-enum-initial-value.AllowExplicitSequentialInitialValues: false, \ +// RUN: }}' + +enum EError { + // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: inital values in enum 'EError' are not consistent + // CHECK-MESSAGES-ENABLE: :[[@LINE-2]]:1: warning: inital values in enum 'EError' are not consistent + EError_a = 1, + EError_b, + // CHECK-FIXES: EError_b = 2, + EError_c = 3, +}; + +enum ENone { + ENone_a, + ENone_b, + EENone_c, +}; + +enum EFirst { + EFirst_a = 1, + EFirst_b, + EFirst_c, +}; + +enum EAll { + EAll_a = 1, + EAll_b = 2, + EAll_c = 4, +}; + +#define ENUMERATOR_1 EMacro1_b +enum EMacro1 { + // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: inital values in enum 'EMacro1' are not consistent + // CHECK-MESSAGES-ENABLE: :[[@LINE-2]]:1: warning: inital values in enum 'EMacro1' are not consistent + EMacro1_a = 1, + ENUMERATOR_1, + // CHECK-FIXES: ENUMERATOR_1 = 2, + EMacro1_c = 3, +}; + + +#define ENUMERATOR_2 EMacro2_b = 2 +enum EMacro2 { + // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: inital values in enum 'EMacro2' are not consistent + // CHECK-MESSAGES-ENABLE: :[[@LINE-2]]:1: warning: inital values in enum 'EMacro2' are not consistent + EMacro2_a = 1, + ENUMERATOR_2, + EMacro2_c, + // CHECK-FIXES: EMacro2_c = 3, +}; + +enum EnumZeroFirstInitialValue { + EnumZeroFirstInitialValue_0 = 0, + // CHECK-MESSAGES-ENABLE: :[[@LINE-1]]:3: warning: zero initial value for the first enumerator in 'EnumZeroFirstInitialValue' can be disregarded + // CHECK-FIXES-ENABLE: EnumZeroFirstInitialValue_0 , + EnumZeroFirstInitialValue_1, + EnumZeroFirstInitialValue_2, +}; + +enum EnumZeroFirstInitialValueWithComment { + EnumZeroFirstInitialValueWithComment_0 = /* == */ 0, + // CHECK-MESSAGES-ENABLE: :[[@LINE-1]]:3: warning: zero initial value for the first enumerator in 'EnumZeroFirstInitialValueWithComment' can be disregarded + // CHECK-FIXES-ENABLE: EnumZeroFirstInitialValueWithComment_0 /* == */ , + EnumZeroFirstInitialValueWithComment_1, + EnumZeroFirstInitialValueWithComment_2, +}; + +enum EnumSequentialInitialValue { + // CHECK-MESSAGES-ENABLE: :[[@LINE-1]]:1: warning: sequential initial value in 'EnumSequentialInitialValue' can be ignored + EnumSequentialInitialValue_0 = 2, + // CHECK-FIXES-ENABLE: EnumSequentialInitialValue_0 = 2, + EnumSequentialInitialValue_1 = 3, + // CHECK-FIXES-ENABLE: EnumSequentialInitialValue_1 , + EnumSequentialInitialValue_2 = 4, + // CHECK-FIXES-ENABLE: EnumSequentialInitialValue_2 , +}; diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.cpp new file mode 100644 index 000000000000..3c4ba970372a --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.cpp @@ -0,0 +1,27 @@ +// RUN: %check_clang_tidy %s readability-enum-initial-value %t + +enum class EError { + // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: inital values in enum 'EError' are not consistent + EError_a = 1, + EError_b, + // CHECK-FIXES: EError_b = 2, + EError_c = 3, +}; + +enum class ENone { + ENone_a, + ENone_b, + EENone_c, +}; + +enum class EFirst { + EFirst_a = 1, + EFirst_b, + EFirst_c, +}; + +enum class EAll { + EAll_a = 1, + EAll_b = 2, + EAll_c = 3, +}; -- GitLab From a4dec9d6bc67c4d8fbd4a4f54ffaa0399def9627 Mon Sep 17 00:00:00 2001 From: Ryotaro KASUGA Date: Mon, 1 Apr 2024 17:04:44 +0900 Subject: [PATCH 0100/1690] [CodeGen] Fix register pressure computation in MachinePipeliner (#87030) `RegisterClassInfo::getRegPressureSetLimit` has been changed to return a smaller value than before so the limit may become negative in later calculations. As a workaround, change to use `TargetRegisterInfo::getRegPressureSetLimit`. Also improve tests. --- llvm/lib/CodeGen/MachinePipeliner.cpp | 2 +- llvm/test/CodeGen/AArch64/sms-regpress.mir | 158 +++++++++++++++++ llvm/test/CodeGen/PowerPC/sms-regpress.mir | 186 +++------------------ 3 files changed, 180 insertions(+), 166 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/sms-regpress.mir diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp index eb42a78603d4..b9c6765be445 100644 --- a/llvm/lib/CodeGen/MachinePipeliner.cpp +++ b/llvm/lib/CodeGen/MachinePipeliner.cpp @@ -1268,7 +1268,7 @@ private: // Calculate the upper limit of each pressure set void computePressureSetLimit(const RegisterClassInfo &RCI) { for (unsigned PSet = 0; PSet < PSetNum; PSet++) - PressureSetLimit[PSet] = RCI.getRegPressureSetLimit(PSet); + PressureSetLimit[PSet] = TRI->getRegPressureSetLimit(MF, PSet); // We assume fixed registers, such as stack pointer, are already in use. // Therefore subtracting the weight of the fixed registers from the limit of diff --git a/llvm/test/CodeGen/AArch64/sms-regpress.mir b/llvm/test/CodeGen/AArch64/sms-regpress.mir new file mode 100644 index 000000000000..ad98d5c6124f --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sms-regpress.mir @@ -0,0 +1,158 @@ +# RUN: llc --verify-machineinstrs -mtriple=aarch64 -o - %s -run-pass pipeliner -aarch64-enable-pipeliner -pipeliner-max-mii=40 -pipeliner-register-pressure -pipeliner-ii-search-range=30 -debug-only=pipeliner 2>&1 | FileCheck %s + +# Check that if the register pressure is too high, the schedule is rejected, II is incremented, and scheduling continues. +# The specific value of II is not important. + +# CHECK: {{^ *}}Try to schedule with {{[0-9]+$}} +# CHECK: {{^ *}}Rejected the schedule because of too high register pressure{{$}} +# CHECK: {{^ *}}Try to schedule with {{[0-9]+$}} +# CHECK: {{^ *}}Schedule Found? 1 (II={{[0-9]+}}){{$}} + +--- | + target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" + + define dso_local double @kernel(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %n) local_unnamed_addr { + entry: + %0 = load double, ptr %a, align 8 + %arrayidx1 = getelementptr inbounds i8, ptr %a, i64 8 + %1 = load double, ptr %arrayidx1, align 8 + %cmp133 = icmp sgt i32 %n, 0 + br i1 %cmp133, label %for.body.preheader, label %for.cond.cleanup + + for.body.preheader: ; preds = %entry + %wide.trip.count = zext nneg i32 %n to i64 + br label %for.body + + for.cond.cleanup: ; preds = %for.body, %entry + %res.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add54, %for.body ] + ret double %res.0.lcssa + + for.body: ; preds = %for.body.preheader, %for.body + %lsr.iv137 = phi i64 [ %wide.trip.count, %for.body.preheader ], [ %lsr.iv.next, %for.body ] + %lsr.iv = phi ptr [ %b, %for.body.preheader ], [ %scevgep, %for.body ] + %res.0135 = phi double [ 0.000000e+00, %for.body.preheader ], [ %add54, %for.body ] + %2 = load double, ptr %lsr.iv, align 8 + %3 = tail call double @llvm.fmuladd.f64(double %0, double %2, double %0) + %4 = tail call double @llvm.fmuladd.f64(double %3, double %2, double %3) + %5 = tail call double @llvm.fmuladd.f64(double %4, double %2, double %4) + %6 = tail call double @llvm.fmuladd.f64(double %5, double %2, double %5) + %7 = tail call double @llvm.fmuladd.f64(double %6, double %2, double %6) + %8 = tail call double @llvm.fmuladd.f64(double %7, double %2, double %7) + %9 = tail call double @llvm.fmuladd.f64(double %8, double %2, double %8) + %10 = tail call double @llvm.fmuladd.f64(double %9, double %2, double %9) + %11 = tail call double @llvm.fmuladd.f64(double %10, double %2, double %10) + %12 = tail call double @llvm.fmuladd.f64(double %11, double %2, double %11) + %13 = tail call double @llvm.fmuladd.f64(double %12, double %2, double %12) + %14 = tail call double @llvm.fmuladd.f64(double %13, double %2, double %13) + %15 = tail call double @llvm.fmuladd.f64(double %14, double %2, double %14) + %16 = tail call double @llvm.fmuladd.f64(double %15, double %2, double %15) + %17 = tail call double @llvm.fmuladd.f64(double %16, double %2, double %16) + %18 = tail call double @llvm.fmuladd.f64(double %17, double %2, double %17) + %add = fadd double %17, %18 + %19 = tail call double @llvm.fmuladd.f64(double %18, double %2, double %add) + %add35 = fadd double %10, %19 + %20 = tail call double @llvm.fmuladd.f64(double %3, double %2, double %add35) + %add38 = fadd double %11, %20 + %21 = tail call double @llvm.fmuladd.f64(double %4, double %2, double %add38) + %add41 = fadd double %12, %21 + %22 = tail call double @llvm.fmuladd.f64(double %5, double %2, double %add41) + %add44 = fadd double %14, %15 + %add45 = fadd double %13, %add44 + %add46 = fadd double %add45, %22 + %23 = tail call double @llvm.fmuladd.f64(double %6, double %2, double %add46) + %mul = fmul double %2, %7 + %mul51 = fmul double %1, %mul + %24 = tail call double @llvm.fmuladd.f64(double %mul51, double %9, double %23) + %25 = tail call double @llvm.fmuladd.f64(double %8, double %1, double %24) + %add54 = fadd double %res.0135, %25 + %scevgep = getelementptr i8, ptr %lsr.iv, i64 8 + %lsr.iv.next = add nsw i64 %lsr.iv137, -1 + %exitcond.not = icmp eq i64 %lsr.iv.next, 0 + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body + } + + declare double @llvm.fmuladd.f64(double, double, double) + +... +--- +name: kernel +tracksRegLiveness: true +liveins: + - { reg: '$x0', virtual-reg: '%10' } + - { reg: '$x1', virtual-reg: '%11' } + - { reg: '$w2', virtual-reg: '%12' } +body: | + bb.0.entry: + successors: %bb.1, %bb.4 + liveins: $x0, $x1, $w2 + + %12:gpr32common = COPY $w2 + %11:gpr64 = COPY $x1 + %10:gpr64common = COPY $x0 + dead $wzr = SUBSWri %12, 1, 0, implicit-def $nzcv + Bcc 10, %bb.1, implicit $nzcv + + bb.4: + %13:fpr64 = FMOVD0 + B %bb.2 + + bb.1.for.body.preheader: + %0:fpr64 = LDRDui %10, 0 :: (load (s64) from %ir.a) + %1:fpr64 = LDRDui %10, 1 :: (load (s64) from %ir.arrayidx1) + %16:gpr32 = ORRWrs $wzr, %12, 0 + %2:gpr64all = SUBREG_TO_REG 0, killed %16, %subreg.sub_32 + %15:fpr64 = FMOVD0 + B %bb.3 + + bb.2.for.cond.cleanup: + %3:fpr64 = PHI %13, %bb.4, %7, %bb.3 + $d0 = COPY %3 + RET_ReallyLR implicit $d0 + + bb.3.for.body: + successors: %bb.2, %bb.3 + + %4:gpr64sp = PHI %2, %bb.1, %9, %bb.3 + %5:gpr64sp = PHI %11, %bb.1, %8, %bb.3 + %6:fpr64 = PHI %15, %bb.1, %7, %bb.3 + early-clobber %17:gpr64sp, %18:fpr64 = LDRDpost %5, 8 :: (load (s64) from %ir.lsr.iv) + %19:fpr64 = nofpexcept FMADDDrrr %0, %18, %0, implicit $fpcr + %20:fpr64 = nofpexcept FMADDDrrr %19, %18, %19, implicit $fpcr + %21:fpr64 = nofpexcept FMADDDrrr %20, %18, %20, implicit $fpcr + %22:fpr64 = nofpexcept FMADDDrrr %21, %18, %21, implicit $fpcr + %23:fpr64 = nofpexcept FMADDDrrr %22, %18, %22, implicit $fpcr + %24:fpr64 = nofpexcept FMADDDrrr %23, %18, %23, implicit $fpcr + %25:fpr64 = nofpexcept FMADDDrrr %24, %18, %24, implicit $fpcr + %26:fpr64 = nofpexcept FMADDDrrr %25, %18, %25, implicit $fpcr + %27:fpr64 = nofpexcept FMADDDrrr %26, %18, %26, implicit $fpcr + %28:fpr64 = nofpexcept FMADDDrrr %27, %18, %27, implicit $fpcr + %29:fpr64 = nofpexcept FMADDDrrr %28, %18, %28, implicit $fpcr + %30:fpr64 = nofpexcept FMADDDrrr %29, %18, %29, implicit $fpcr + %31:fpr64 = nofpexcept FMADDDrrr %30, %18, %30, implicit $fpcr + %32:fpr64 = nofpexcept FMADDDrrr %31, %18, %31, implicit $fpcr + %33:fpr64 = nofpexcept FMADDDrrr %32, %18, %32, implicit $fpcr + %34:fpr64 = nofpexcept FMADDDrrr %33, %18, %33, implicit $fpcr + %35:fpr64 = nofpexcept FADDDrr %33, %34, implicit $fpcr + %36:fpr64 = nofpexcept FMADDDrrr %34, %18, killed %35, implicit $fpcr + %37:fpr64 = nofpexcept FADDDrr %26, killed %36, implicit $fpcr + %38:fpr64 = nofpexcept FMADDDrrr %19, %18, killed %37, implicit $fpcr + %39:fpr64 = nofpexcept FADDDrr %27, killed %38, implicit $fpcr + %40:fpr64 = nofpexcept FMADDDrrr %20, %18, killed %39, implicit $fpcr + %41:fpr64 = nofpexcept FADDDrr %28, killed %40, implicit $fpcr + %42:fpr64 = nofpexcept FMADDDrrr %21, %18, killed %41, implicit $fpcr + %43:fpr64 = nofpexcept FADDDrr %30, %31, implicit $fpcr + %44:fpr64 = nofpexcept FADDDrr %29, killed %43, implicit $fpcr + %45:fpr64 = nofpexcept FADDDrr killed %44, killed %42, implicit $fpcr + %46:fpr64 = nofpexcept FMADDDrrr %22, %18, killed %45, implicit $fpcr + %47:fpr64 = nofpexcept FMULDrr %18, %23, implicit $fpcr + %48:fpr64 = nofpexcept FMULDrr %1, killed %47, implicit $fpcr + %49:fpr64 = nofpexcept FMADDDrrr killed %48, %25, killed %46, implicit $fpcr + %50:fpr64 = nofpexcept FMADDDrrr %24, %1, killed %49, implicit $fpcr + %7:fpr64 = nofpexcept FADDDrr %6, killed %50, implicit $fpcr + %8:gpr64all = COPY %17 + %51:gpr64 = nsw SUBSXri %4, 1, 0, implicit-def $nzcv + %9:gpr64all = COPY %51 + Bcc 0, %bb.2, implicit $nzcv + B %bb.3 + +... diff --git a/llvm/test/CodeGen/PowerPC/sms-regpress.mir b/llvm/test/CodeGen/PowerPC/sms-regpress.mir index cebd78af882d..b01115c49fd8 100644 --- a/llvm/test/CodeGen/PowerPC/sms-regpress.mir +++ b/llvm/test/CodeGen/PowerPC/sms-regpress.mir @@ -1,41 +1,30 @@ -# RUN: llc --verify-machineinstrs -mcpu=pwr9 -o - %s -run-pass=pipeliner -ppc-enable-pipeliner -pipeliner-register-pressure -pipeliner-max-mii=50 -pipeliner-ii-search-range=30 -pipeliner-max-stages=10 -debug-only=pipeliner 2>&1 | FileCheck %s +# RUN: llc --verify-machineinstrs -mcpu=pwr9 -o - %s -run-pass=pipeliner -ppc-enable-pipeliner -pipeliner-register-pressure -pipeliner-max-mii=50 -pipeliner-ii-search-range=30 -pipeliner-max-stages=10 -debug-only=pipeliner 2>&1 | FileCheck %s # REQUIRES: asserts # Check that if the register pressure is too high, the schedule is rejected, II is incremented, and scheduling continues. # The specific value of II is not important. -# CHECK: Try to schedule with 21 -# CHECK: Can't schedule -# CHECK: Try to schedule with 22 -# CHECK: Can't schedule -# CHECK: Try to schedule with 23 -# CHECK: Rejected the schedule because of too high register pressure -# CHECK: Try to schedule with 24 -# CHECK: Rejected the schedule because of too high register pressure -# CHECK: Try to schedule with 25 -# CHECK: Rejected the schedule because of too high register pressure -# CHECK: Try to schedule with 26 -# CHECK: Schedule Found? 1 (II=26) +# CHECK: {{^ *}}Try to schedule with {{[0-9]+$}} +# CHECK: {{^ *}}Rejected the schedule because of too high register pressure{{$}} +# CHECK: {{^ *}}Try to schedule with {{[0-9]+$}} +# CHECK: {{^ *}}Schedule Found? 1 (II={{[0-9]+}}){{$}} --- | - ; ModuleID = 'a.ll' - source_filename = "a.c" target datalayout = "e-m:e-Fn32-i64:64-n32:64" target triple = "ppc64le" - ; Function Attrs: nofree nosync nounwind memory(argmem: read) uwtable - define dso_local double @kernel(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef signext %n) local_unnamed_addr #0 { + define dso_local double @kernel(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef signext %n) local_unnamed_addr { entry: - %0 = load double, ptr %a, align 8, !tbaa !3 - %arrayidx1 = getelementptr inbounds double, ptr %a, i64 1 - %1 = load double, ptr %arrayidx1, align 8, !tbaa !3 + %0 = load double, ptr %a, align 8 + %arrayidx1 = getelementptr inbounds i8, ptr %a, i64 8 + %1 = load double, ptr %arrayidx1, align 8 %cmp163 = icmp sgt i32 %n, 0 br i1 %cmp163, label %for.body.preheader, label %for.cond.cleanup for.body.preheader: ; preds = %entry - %wide.trip.count = zext i32 %n to i64 - %scevgep1 = getelementptr i8, ptr %b, i64 -8 + %wide.trip.count = zext nneg i32 %n to i64 + %scevgep167 = getelementptr i8, ptr %b, i64 -8 call void @llvm.set.loop.iterations.i64(i64 %wide.trip.count) br label %for.body @@ -43,11 +32,11 @@ %res.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %30, %for.body ] ret double %res.0.lcssa - for.body: ; preds = %for.body, %for.body.preheader + for.body: ; preds = %for.body.preheader, %for.body %res.0165 = phi double [ 0.000000e+00, %for.body.preheader ], [ %30, %for.body ] - %2 = phi ptr [ %scevgep1, %for.body.preheader ], [ %3, %for.body ] + %2 = phi ptr [ %scevgep167, %for.body.preheader ], [ %3, %for.body ] %3 = getelementptr i8, ptr %2, i64 8 - %4 = load double, ptr %3, align 8, !tbaa !3 + %4 = load double, ptr %3, align 8 %5 = tail call double @llvm.fmuladd.f64(double %0, double %4, double %0) %6 = tail call double @llvm.fmuladd.f64(double %5, double %4, double %5) %7 = tail call double @llvm.fmuladd.f64(double %6, double %4, double %6) @@ -92,152 +81,23 @@ %mul66 = fmul double %12, %mul65 %30 = tail call double @llvm.fmuladd.f64(double %mul66, double %10, double %res.0165) %31 = call i1 @llvm.loop.decrement.i64(i64 1) - br i1 %31, label %for.body, label %for.cond.cleanup, !llvm.loop !7 + br i1 %31, label %for.body, label %for.cond.cleanup } - ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) - declare double @llvm.fmuladd.f64(double, double, double) #1 + declare double @llvm.fmuladd.f64(double, double, double) - ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn - declare void @llvm.set.loop.iterations.i64(i64) #2 + declare void @llvm.set.loop.iterations.i64(i64) - ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn - declare i1 @llvm.loop.decrement.i64(i64) #2 + declare i1 @llvm.loop.decrement.i64(i64) - attributes #0 = { nofree nosync nounwind memory(argmem: read) uwtable "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crbits,+crypto,+direct-move,+extdiv,+htm,+isa-v206-instructions,+isa-v207-instructions,+isa-v30-instructions,+power8-vector,+power9-vector,+quadword-atomics,+vsx,-aix-small-local-exec-tls,-privileged,-rop-protect,-spe" } - attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } - attributes #2 = { nocallback noduplicate nofree nosync nounwind willreturn } - - !llvm.module.flags = !{!0, !1} - !llvm.ident = !{!2} - - !0 = !{i32 1, !"wchar_size", i32 4} - !1 = !{i32 7, !"uwtable", i32 2} - !2 = !{!"clang version 18.0.0 (https://miratech-soft@dev.azure.com/miratech-soft/llvm/_git/llvm c8d01fb665fc5d9378100a6d92ebcd3be49be655)"} - !3 = !{!4, !4, i64 0} - !4 = !{!"double", !5, i64 0} - !5 = !{!"omnipotent char", !6, i64 0} - !6 = !{!"Simple C/C++ TBAA"} - !7 = distinct !{!7, !8, !9} - !8 = !{!"llvm.loop.mustprogress"} - !9 = !{!"llvm.loop.unroll.disable"} - ... --- name: kernel -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false tracksRegLiveness: true -hasWinCFI: false -callsEHReturn: false -callsUnwindInit: false -hasEHCatchret: false -hasEHScopes: false -hasEHFunclets: false -isOutlined: false -debugInstrRef: false -failsVerification: false -tracksDebugUserValues: false -registers: - - { id: 0, class: vsfrc, preferred-register: '' } - - { id: 1, class: vsfrc, preferred-register: '' } - - { id: 2, class: g8rc, preferred-register: '' } - - { id: 3, class: vsfrc, preferred-register: '' } - - { id: 4, class: vsfrc, preferred-register: '' } - - { id: 5, class: g8rc_and_g8rc_nox0, preferred-register: '' } - - { id: 6, class: g8rc, preferred-register: '' } - - { id: 7, class: vsfrc, preferred-register: '' } - - { id: 8, class: g8rc_and_g8rc_nox0, preferred-register: '' } - - { id: 9, class: g8rc_and_g8rc_nox0, preferred-register: '' } - - { id: 10, class: g8rc, preferred-register: '' } - - { id: 11, class: gprc, preferred-register: '' } - - { id: 12, class: vsfrc, preferred-register: '' } - - { id: 13, class: crrc, preferred-register: '' } - - { id: 14, class: vsfrc, preferred-register: '' } - - { id: 15, class: g8rc, preferred-register: '' } - - { id: 16, class: g8rc, preferred-register: '' } - - { id: 17, class: g8rc, preferred-register: '' } - - { id: 18, class: f8rc, preferred-register: '' } - - { id: 19, class: g8rc_and_g8rc_nox0, preferred-register: '' } - - { id: 20, class: vsfrc, preferred-register: '' } - - { id: 21, class: vsfrc, preferred-register: '' } - - { id: 22, class: vsfrc, preferred-register: '' } - - { id: 23, class: vsfrc, preferred-register: '' } - - { id: 24, class: vsfrc, preferred-register: '' } - - { id: 25, class: vsfrc, preferred-register: '' } - - { id: 26, class: vsfrc, preferred-register: '' } - - { id: 27, class: vsfrc, preferred-register: '' } - - { id: 28, class: vsfrc, preferred-register: '' } - - { id: 29, class: vsfrc, preferred-register: '' } - - { id: 30, class: vsfrc, preferred-register: '' } - - { id: 31, class: vsfrc, preferred-register: '' } - - { id: 32, class: vsfrc, preferred-register: '' } - - { id: 33, class: vsfrc, preferred-register: '' } - - { id: 34, class: vsfrc, preferred-register: '' } - - { id: 35, class: vsfrc, preferred-register: '' } - - { id: 36, class: vsfrc, preferred-register: '' } - - { id: 37, class: vsfrc, preferred-register: '' } - - { id: 38, class: vsfrc, preferred-register: '' } - - { id: 39, class: vsfrc, preferred-register: '' } - - { id: 40, class: vsfrc, preferred-register: '' } - - { id: 41, class: vsfrc, preferred-register: '' } - - { id: 42, class: vsfrc, preferred-register: '' } - - { id: 43, class: vsfrc, preferred-register: '' } - - { id: 44, class: vsfrc, preferred-register: '' } - - { id: 45, class: vsfrc, preferred-register: '' } - - { id: 46, class: vsfrc, preferred-register: '' } - - { id: 47, class: vsfrc, preferred-register: '' } - - { id: 48, class: vsfrc, preferred-register: '' } - - { id: 49, class: vsfrc, preferred-register: '' } - - { id: 50, class: vsfrc, preferred-register: '' } - - { id: 51, class: vsfrc, preferred-register: '' } - - { id: 52, class: vsfrc, preferred-register: '' } - - { id: 53, class: vsfrc, preferred-register: '' } - - { id: 54, class: vsfrc, preferred-register: '' } - - { id: 55, class: vsfrc, preferred-register: '' } - - { id: 56, class: vsfrc, preferred-register: '' } - - { id: 57, class: vsfrc, preferred-register: '' } - - { id: 58, class: vsfrc, preferred-register: '' } - - { id: 59, class: vsfrc, preferred-register: '' } - - { id: 60, class: vsfrc, preferred-register: '' } - - { id: 61, class: vsfrc, preferred-register: '' } - - { id: 62, class: crbitrc, preferred-register: '' } liveins: - { reg: '$x3', virtual-reg: '%8' } - { reg: '$x4', virtual-reg: '%9' } - { reg: '$x5', virtual-reg: '%10' } -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 1 - adjustsStack: false - hasCalls: false - stackProtector: '' - functionContext: '' - maxCallFrameSize: 4294967295 - cvBytesOfCalleeSavedRegisters: 0 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - hasTailCall: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: [] -stack: [] -entry_values: [] -callSites: [] -debugValueSubstitutions: [] -constants: [] -machineFunctionInfo: {} body: | bb.0.entry: successors: %bb.2(0x50000000), %bb.1(0x30000000) @@ -251,16 +111,12 @@ body: | BCC 44, killed %13, %bb.2 bb.1: - successors: %bb.3(0x80000000) - %12:vsfrc = XXLXORdpz B %bb.3 bb.2.for.body.preheader: - successors: %bb.4(0x80000000) - - %0:vsfrc = DFLOADf64 0, %8 :: (load (s64) from %ir.a, !tbaa !3) - %1:vsfrc = DFLOADf64 8, killed %8 :: (load (s64) from %ir.arrayidx1, !tbaa !3) + %0:vsfrc = DFLOADf64 0, %8 :: (load (s64) from %ir.a) + %1:vsfrc = DFLOADf64 8, killed %8 :: (load (s64) from %ir.arrayidx1) %16:g8rc = IMPLICIT_DEF %15:g8rc = INSERT_SUBREG killed %16, killed %11, %subreg.sub_32 %17:g8rc = RLDICL killed %15, 0, 32 @@ -279,7 +135,7 @@ body: | %4:vsfrc = PHI %14, %bb.2, %7, %bb.4 %5:g8rc_and_g8rc_nox0 = PHI %2, %bb.2, %6, %bb.4 - %18:f8rc, %19:g8rc_and_g8rc_nox0 = LFDU 8, killed %5 :: (load (s64) from %ir.3, !tbaa !3) + %18:f8rc, %19:g8rc_and_g8rc_nox0 = LFDU 8, killed %5 :: (load (s64) from %ir.3) %6:g8rc = COPY killed %19 %20:vsfrc = nofpexcept XSMADDADP %0, %0, %18, implicit $rm %21:vsfrc = nofpexcept XSMADDADP %20, %20, %18, implicit $rm -- GitLab From ef0291e5f4451abbafab0c839bf51a6382f735f3 Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Mon, 1 Apr 2024 16:10:10 +0800 Subject: [PATCH 0101/1690] [NFC] [Serialization] Reordering lexcical and visible TU block after type decl offsets This patch reorder the lexical block for the translation unit, visible update block for the TU and the viisble upaete block for the extern C context after the type decl offsets block. This should be a NFC patch. This is helpful for later optimizations for eliding unreachable declarations in the global module fragment. See the comments in https://github.com/llvm/llvm-project/pull/76930. Simply, if we want to get the reachable sets of declaratins during the writing process, we need to write the file-level context later than the process of writing declarations (which is the main process to determine the reachable set). --- clang/lib/Serialization/ASTWriter.cpp | 71 ++++++++++++++---------- clang/test/Modules/language-linkage.cppm | 2 +- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 1e5734c9c834..2438fbc16606 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -4959,38 +4959,12 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, Stream.EmitRecord(METADATA_OLD_FORMAT, Record); } - // Create a lexical update block containing all of the declarations in the - // translation unit that do not come from other AST files. const TranslationUnitDecl *TU = Context.getTranslationUnitDecl(); - SmallVector NewGlobalKindDeclPairs; - for (const auto *D : TU->noload_decls()) { - if (!D->isFromASTFile()) { - NewGlobalKindDeclPairs.push_back(D->getKind()); - NewGlobalKindDeclPairs.push_back(GetDeclRef(D)); - } - } - - auto Abv = std::make_shared(); - Abv->Add(llvm::BitCodeAbbrevOp(TU_UPDATE_LEXICAL)); - Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob)); - unsigned TuUpdateLexicalAbbrev = Stream.EmitAbbrev(std::move(Abv)); - { - RecordData::value_type Record[] = {TU_UPDATE_LEXICAL}; - Stream.EmitRecordWithBlob(TuUpdateLexicalAbbrev, Record, - bytes(NewGlobalKindDeclPairs)); - } - // And a visible updates block for the translation unit. - Abv = std::make_shared(); - Abv->Add(llvm::BitCodeAbbrevOp(UPDATE_VISIBLE)); - Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); - Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob)); - UpdateVisibleAbbrev = Stream.EmitAbbrev(std::move(Abv)); - WriteDeclContextVisibleUpdate(TU); - - // If we have any extern "C" names, write out a visible update for them. - if (Context.ExternCContext) - WriteDeclContextVisibleUpdate(Context.ExternCContext); + // Force all top level declarations to be emitted. + for (const auto *D : TU->noload_decls()) + if (!D->isFromASTFile()) + GetDeclRef(D); // If the translation unit has an anonymous namespace, and we don't already // have an update block for it, write it as an update block. @@ -5131,6 +5105,14 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, for (auto *D : SemaRef.DeclsToCheckForDeferredDiags) DeclsToCheckForDeferredDiags.push_back(GetDeclRef(D)); + { + auto Abv = std::make_shared(); + Abv->Add(llvm::BitCodeAbbrevOp(UPDATE_VISIBLE)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob)); + UpdateVisibleAbbrev = Stream.EmitAbbrev(std::move(Abv)); + } + RecordData DeclUpdatesOffsetsRecord; // Keep writing types, declarations, and declaration update records @@ -5158,6 +5140,35 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, WriteTypeDeclOffsets(); if (!DeclUpdatesOffsetsRecord.empty()) Stream.EmitRecord(DECL_UPDATE_OFFSETS, DeclUpdatesOffsetsRecord); + + // Create a lexical update block containing all of the declarations in the + // translation unit that do not come from other AST files. + { + SmallVector NewGlobalKindDeclPairs; + for (const auto *D : TU->noload_decls()) { + if (!D->isFromASTFile()) { + NewGlobalKindDeclPairs.push_back(D->getKind()); + NewGlobalKindDeclPairs.push_back(GetDeclRef(D)); + } + } + + auto Abv = std::make_shared(); + Abv->Add(llvm::BitCodeAbbrevOp(TU_UPDATE_LEXICAL)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob)); + unsigned TuUpdateLexicalAbbrev = Stream.EmitAbbrev(std::move(Abv)); + + RecordData::value_type Record[] = {TU_UPDATE_LEXICAL}; + Stream.EmitRecordWithBlob(TuUpdateLexicalAbbrev, Record, + bytes(NewGlobalKindDeclPairs)); + } + + // And a visible updates block for the translation unit. + WriteDeclContextVisibleUpdate(TU); + + // If we have any extern "C" names, write out a visible update for them. + if (Context.ExternCContext) + WriteDeclContextVisibleUpdate(Context.ExternCContext); + WriteFileDeclIDsMap(); WriteSourceManagerBlock(Context.getSourceManager(), PP); WriteComments(); diff --git a/clang/test/Modules/language-linkage.cppm b/clang/test/Modules/language-linkage.cppm index bf7982cd9207..a5db9e9ebc07 100644 --- a/clang/test/Modules/language-linkage.cppm +++ b/clang/test/Modules/language-linkage.cppm @@ -14,5 +14,5 @@ void foo() {} extern "C" void bar() {} -// CHECK: define {{.*}}@bar( // CHECK: define {{.*}}@_Z3foov( +// CHECK: define {{.*}}@bar( -- GitLab From da1d3d8fb9e7dba1cc89327f5119fa7c0cadef81 Mon Sep 17 00:00:00 2001 From: superZWT123 Date: Mon, 1 Apr 2024 17:19:46 +0800 Subject: [PATCH 0102/1690] =?UTF-8?q?[TableGen]=20Introduce=20a=20less=20a?= =?UTF-8?q?ggressive=20suppression=20for=20HwMode=20Decoder=E2=80=A6=20(#8?= =?UTF-8?q?6060)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Remove 'AllModes' and 'DefaultMode' suffixes for DecoderTables under default HwMode. 2. Introduce a less aggressive suppression for HwMode DecoderTable, only reduce necessary tables duplications. This allows encodings under different HwModes to retain the original DecoderNamespace. 3. Change 'suppress-per-hwmode-duplicates' command option from bool type to enum type, allowing users to choose what level of suppression to use. --- llvm/test/TableGen/HwModeEncodeDecode2.td | 9 +- llvm/test/TableGen/HwModeEncodeDecode3.td | 71 ++++++++++----- llvm/utils/TableGen/DecoderEmitter.cpp | 104 +++++++++++++++++----- 3 files changed, 137 insertions(+), 47 deletions(-) diff --git a/llvm/test/TableGen/HwModeEncodeDecode2.td b/llvm/test/TableGen/HwModeEncodeDecode2.td index 5159501d8148..cf96dda6c8bf 100644 --- a/llvm/test/TableGen/HwModeEncodeDecode2.td +++ b/llvm/test/TableGen/HwModeEncodeDecode2.td @@ -1,6 +1,6 @@ // RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | \ // RUN: FileCheck %s --check-prefix=DECODER -// RUN: llvm-tblgen -gen-disassembler --suppress-per-hwmode-duplicates -I \ +// RUN: llvm-tblgen -gen-disassembler --suppress-per-hwmode-duplicates=O2 -I \ // RUN: %p/../../include %s | FileCheck %s --check-prefix=DECODER-SUPPRESS // Test duplicate table suppression for per-HwMode decoders. @@ -105,11 +105,10 @@ let OutOperandList = (outs) in { // DECODER-DAG: Opcode: fooTypeEncA:baz // DECODER-DAG: Opcode: bar - -// DECODER-SUPPRESS-LABEL: DecoderTableAlt_AllModes32[] = -// DECODER-SUPPRESS-DAG: Opcode: unrelated -// DECODER-SUPPRESS-LABEL: DecoderTable_AllModes32[] = +// DECODER-SUPPRESS-LABEL: DecoderTable32[] = // DECODER-SUPPRESS-DAG: Opcode: bar +// DECODER-SUPPRESS-LABEL: DecoderTableAlt32[] = +// DECODER-SUPPRESS-DAG: Opcode: unrelated // DECODER-SUPPRESS-LABEL: DecoderTable_ModeA32[] = // DECODER-SUPPRESS-DAG: Opcode: fooTypeEncA:foo // DECODER-SUPPRESS-NOT: Opcode: bar diff --git a/llvm/test/TableGen/HwModeEncodeDecode3.td b/llvm/test/TableGen/HwModeEncodeDecode3.td index 406e52d25be7..8e0266b2c55a 100644 --- a/llvm/test/TableGen/HwModeEncodeDecode3.td +++ b/llvm/test/TableGen/HwModeEncodeDecode3.td @@ -2,8 +2,10 @@ // RUN: FileCheck %s --check-prefix=ENCODER // RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | \ // RUN: FileCheck %s --check-prefix=DECODER -// RUN: llvm-tblgen -gen-disassembler --suppress-per-hwmode-duplicates -I \ -// RUN: %p/../../include %s | FileCheck %s --check-prefix=DECODER-SUPPRESS +// RUN: llvm-tblgen -gen-disassembler --suppress-per-hwmode-duplicates=O1 -I \ +// RUN: %p/../../include %s | FileCheck %s --check-prefix=DECODER-SUPPRESS-O1 +// RUN: llvm-tblgen -gen-disassembler --suppress-per-hwmode-duplicates=O2 -I \ +// RUN: %p/../../include %s | FileCheck %s --check-prefix=DECODER-SUPPRESS-O2 include "llvm/Target/Target.td" @@ -99,16 +101,20 @@ def unrelated: Instruction { } -// DECODER-LABEL: DecoderTableAlt_DefaultMode32[] = +// Under default settings, using 'HwMode' to dictate instruction encodings results in +// significant duplication of DecoderTables. The three tables ‘DecoderTableAlt32’, +// ‘DecoderTableAlt_ModeA32’, and ‘DecoderTableAlt_ModeB32’ are exact duplicates and +// could effectively be merged into one. +// DECODER-LABEL: DecoderTable32[] = +// DECODER-DAG: Opcode: bar +// DECODER-LABEL: DecoderTable64[] = +// DECODER-DAG: Opcode: fooTypeEncDefault:foo +// DECODER-LABEL: DecoderTableAlt32[] = // DECODER-DAG: Opcode: unrelated // DECODER-LABEL: DecoderTableAlt_ModeA32[] = // DECODER-DAG: Opcode: unrelated // DECODER-LABEL: DecoderTableAlt_ModeB32[] = // DECODER-DAG: Opcode: unrelated -// DECODER-LABEL: DecoderTable_DefaultMode32[] = -// DECODER-DAG: Opcode: bar -// DECODER-LABEL: DecoderTable_DefaultMode64[] = -// DECODER-DAG: Opcode: fooTypeEncDefault:foo // DECODER-LABEL: DecoderTable_ModeA32[] = // DECODER-DAG: Opcode: fooTypeEncA:foo // DECODER-DAG: Opcode: bar @@ -117,21 +123,42 @@ def unrelated: Instruction { // DECODER-DAG: Opcode: fooTypeEncA:baz // DECODER-DAG: Opcode: bar - -// DECODER-SUPPRESS-LABEL: DecoderTableAlt_AllModes32[] = -// DECODER-SUPPRESS-DAG: Opcode: unrelated -// DECODER-SUPPRESS-LABEL: DecoderTable_AllModes32[] = -// DECODER-SUPPRESS-DAG: Opcode: bar -// DECODER-SUPPRESS-LABEL: DecoderTable_DefaultMode64[] = -// DECODER-SUPPRESS-NOT: Opcode: bar -// DECODER-SUPPRESS-DAG: Opcode: fooTypeEncDefault:foo -// DECODER-SUPPRESS-LABEL: DecoderTable_ModeA32[] = -// DECODER-SUPPRESS-DAG: Opcode: fooTypeEncA:foo -// DECODER-SUPPRESS-NOT: Opcode: bar -// DECODER-SUPPRESS-LABEL: DecoderTable_ModeB32[] = -// DECODER-SUPPRESS-DAG: Opcode: fooTypeEncB:foo -// DECODER-SUPPRESS-DAG: Opcode: fooTypeEncA:baz -// DECODER-SUPPRESS-NOT: Opcode: bar +// Under the 'O1' optimization level, unnecessary duplicate tables will be eliminated, +// reducing the three ‘Alt’ tables down to just one. +// DECODER-SUPPRESS-O1-LABEL: DecoderTable32[] = +// DECODER-SUPPRESS-O1-DAG: Opcode: bar +// DECODER-SUPPRESS-O1-LABEL: DecoderTable64[] = +// DECODER-SUPPRESS-O1-DAG: Opcode: fooTypeEncDefault:foo +// DECODER-SUPPRESS-O1-LABEL: DecoderTableAlt32[] = +// DECODER-SUPPRESS-O1-DAG: Opcode: unrelated +// DECODER-SUPPRESS-O1-LABEL: DecoderTable_ModeA32[] = +// DECODER-SUPPRESS-O1-DAG: Opcode: fooTypeEncA:foo +// DECODER-SUPPRESS-O1-DAG: Opcode: bar +// DECODER-SUPPRESS-O1-LABEL: DecoderTable_ModeB32[] = +// DECODER-SUPPRESS-O1-DAG: Opcode: fooTypeEncB:foo +// DECODER-SUPPRESS-O1-DAG: Opcode: fooTypeEncA:baz +// DECODER-SUPPRESS-O1-DAG: Opcode: bar + +// Under the 'O2' optimization condition, instructions possessing the 'EncodingByHwMode' +// attribute will be extracted from their original DecoderNamespace and placed into their +// respective HwMode tables. Meanwhile, other instructions that do not have the 'EncodingByHwMode' +// attribute but are within the same DecoderNamespace will be stored in the 'Default' table. This +// approach will significantly reduce instruction redundancy, but it necessitates users to thoroughly +// consider the interplay between HwMode and DecoderNamespace for their instructions. +// DECODER-SUPPRESS-O2-LABEL: DecoderTable32[] = +// DECODER-SUPPRESS-O2-DAG: Opcode: bar +// DECODER-SUPPRESS-O2-LABEL: DecoderTable64[] = +// DECODER-SUPPRESS-O2-NOT: Opcode: bar +// DECODER-SUPPRESS-O2-DAG: Opcode: fooTypeEncDefault:foo +// DECODER-SUPPRESS-O2-LABEL: DecoderTableAlt32[] = +// DECODER-SUPPRESS-O2-DAG: Opcode: unrelated +// DECODER-SUPPRESS-O2-LABEL: DecoderTable_ModeA32[] = +// DECODER-SUPPRESS-O2-DAG: Opcode: fooTypeEncA:foo +// DECODER-SUPPRESS-O2-NOT: Opcode: bar +// DECODER-SUPPRESS-O2-LABEL: DecoderTable_ModeB32[] = +// DECODER-SUPPRESS-O2-DAG: Opcode: fooTypeEncB:foo +// DECODER-SUPPRESS-O2-DAG: Opcode: fooTypeEncA:baz +// DECODER-SUPPRESS-O2-NOT: Opcode: bar // ENCODER-LABEL: static const uint64_t InstBits_DefaultMode[] = { // ENCODER: UINT64_C(2), // bar diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp index 494dc93faace..3bd7f432ff9a 100644 --- a/llvm/utils/TableGen/DecoderEmitter.cpp +++ b/llvm/utils/TableGen/DecoderEmitter.cpp @@ -54,10 +54,27 @@ using namespace llvm; extern cl::OptionCategory DisassemblerEmitterCat; -cl::opt DecoderEmitterSuppressDuplicates( +enum SuppressLevel { + SUPPRESSION_DISABLE, + SUPPRESSION_LEVEL1, + SUPPRESSION_LEVEL2 +}; + +cl::opt DecoderEmitterSuppressDuplicates( "suppress-per-hwmode-duplicates", cl::desc("Suppress duplication of instrs into per-HwMode decoder tables"), - cl::init(false), cl::cat(DisassemblerEmitterCat)); + cl::values( + clEnumValN( + SUPPRESSION_DISABLE, "O0", + "Do not prevent DecoderTable duplications caused by HwModes"), + clEnumValN( + SUPPRESSION_LEVEL1, "O1", + "Remove duplicate DecoderTable entries generated due to HwModes"), + clEnumValN( + SUPPRESSION_LEVEL2, "O2", + "Extract HwModes-specific instructions into new DecoderTables, " + "significantly reducing Table Duplications")), + cl::init(SUPPRESSION_DISABLE), cl::cat(DisassemblerEmitterCat)); namespace { @@ -128,6 +145,7 @@ struct EncodingIDAndOpcode { }; using EncodingIDsVec = std::vector; +using NamespacesHwModesMap = std::map>; raw_ostream &operator<<(raw_ostream &OS, const EncodingAndInst &Value) { if (Value.EncodingDef != Value.Inst->TheDef) @@ -2417,21 +2435,65 @@ static bool Check(DecodeStatus &Out, DecodeStatus In) { // Collect all HwModes referenced by the target for encoding purposes, // returning a vector of corresponding names. -static void -collectHwModesReferencedForEncodings(const CodeGenHwModes &HWM, - std::vector &Names) { +static void collectHwModesReferencedForEncodings( + const CodeGenHwModes &HWM, std::vector &Names, + NamespacesHwModesMap &NamespacesWithHwModes) { SmallBitVector BV(HWM.getNumModeIds()); for (const auto &MS : HWM.getHwModeSelects()) { for (const HwModeSelect::PairType &P : MS.second.Items) { - if (P.second->isSubClassOf("InstructionEncoding")) + if (P.second->isSubClassOf("InstructionEncoding")) { + std::string DecoderNamespace = + std::string(P.second->getValueAsString("DecoderNamespace")); + if (P.first == DefaultMode) { + NamespacesWithHwModes[DecoderNamespace].insert(""); + } else { + NamespacesWithHwModes[DecoderNamespace].insert( + HWM.getMode(P.first).Name); + } BV.set(P.first); + } } } transform(BV.set_bits(), std::back_inserter(Names), [&HWM](const int &M) { + if (M == DefaultMode) + return StringRef(""); return HWM.getModeName(M, /*IncludeDefault=*/true); }); } +static void +handleHwModesUnrelatedEncodings(const CodeGenInstruction *Instr, + const std::vector &HwModeNames, + NamespacesHwModesMap &NamespacesWithHwModes, + std::vector &GlobalEncodings) { + const Record *InstDef = Instr->TheDef; + + switch (DecoderEmitterSuppressDuplicates) { + case SUPPRESSION_DISABLE: { + for (StringRef HwModeName : HwModeNames) + GlobalEncodings.emplace_back(InstDef, Instr, HwModeName); + break; + } + case SUPPRESSION_LEVEL1: { + std::string DecoderNamespace = + std::string(InstDef->getValueAsString("DecoderNamespace")); + auto It = NamespacesWithHwModes.find(DecoderNamespace); + if (It != NamespacesWithHwModes.end()) { + for (StringRef HwModeName : It->second) + GlobalEncodings.emplace_back(InstDef, Instr, HwModeName); + } else { + // Only emit the encoding once, as it's DecoderNamespace doesn't + // contain any HwModes. + GlobalEncodings.emplace_back(InstDef, Instr, ""); + } + break; + } + case SUPPRESSION_LEVEL2: + GlobalEncodings.emplace_back(InstDef, Instr, ""); + break; + } +} + // Emits disassembler code for instruction decoding. void DecoderEmitter::run(raw_ostream &o) { formatted_raw_ostream OS(o); @@ -2457,10 +2519,12 @@ namespace llvm { // Parameterize the decoders based on namespace and instruction width. // First, collect all encoding-related HwModes referenced by the target. + // And establish a mapping table between DecoderNamespace and HwMode. // If HwModeNames is empty, add the empty string so we always have one HwMode. const CodeGenHwModes &HWM = Target.getHwModes(); std::vector HwModeNames; - collectHwModesReferencedForEncodings(HWM, HwModeNames); + NamespacesHwModesMap NamespacesWithHwModes; + collectHwModesReferencedForEncodings(HWM, HwModeNames, NamespacesWithHwModes); if (HwModeNames.empty()) HwModeNames.push_back(""); @@ -2471,22 +2535,22 @@ namespace llvm { if (const RecordVal *RV = InstDef->getValue("EncodingInfos")) { if (DefInit *DI = dyn_cast_or_null(RV->getValue())) { EncodingInfoByHwMode EBM(DI->getDef(), HWM); - for (auto &KV : EBM) - NumberedEncodings.emplace_back( - KV.second, NumberedInstruction, - HWM.getModeName(KV.first, /*IncludeDefault=*/true)); + for (auto &[ModeId, Encoding] : EBM) { + // DecoderTables with DefaultMode should not have any suffix. + if (ModeId == DefaultMode) { + NumberedEncodings.emplace_back(Encoding, NumberedInstruction, ""); + } else { + NumberedEncodings.emplace_back(Encoding, NumberedInstruction, + HWM.getMode(ModeId).Name); + } + } continue; } } - // This instruction is encoded the same on all HwModes. Emit it for all - // HwModes by default, otherwise leave it in a single common table. - if (DecoderEmitterSuppressDuplicates) { - NumberedEncodings.emplace_back(InstDef, NumberedInstruction, "AllModes"); - } else { - for (StringRef HwModeName : HwModeNames) - NumberedEncodings.emplace_back(InstDef, NumberedInstruction, - HwModeName); - } + // This instruction is encoded the same on all HwModes. + // According to user needs, provide varying degrees of suppression. + handleHwModesUnrelatedEncodings(NumberedInstruction, HwModeNames, + NamespacesWithHwModes, NumberedEncodings); } for (const auto &NumberedAlias : RK.getAllDerivedDefinitions("AdditionalEncoding")) -- GitLab From 8827ff92b96d78ef455157574061d745df2909af Mon Sep 17 00:00:00 2001 From: Victor Perez Date: Mon, 1 Apr 2024 11:57:14 +0200 Subject: [PATCH 0103/1690] [MLIR][Arith] Add rounding mode attribute to `truncf` (#86152) Add rounding mode attribute to `arith`. This attribute can be used in different FP `arith` operations to control rounding mode. Rounding modes correspond to IEEE 754-specified rounding modes. Use in `arith.truncf` folding. As this is not supported in dialects other than LLVM, conversion should fail for now in case this attribute is present. --------- Signed-off-by: Victor Perez --- .../ArithCommon/AttrToLLVMConverter.h | 48 +++++++++++++++++++ .../mlir/Dialect/Arith/IR/ArithBase.td | 25 ++++++++++ .../include/mlir/Dialect/Arith/IR/ArithOps.td | 21 ++++++-- .../Dialect/Arith/IR/ArithOpsInterfaces.td | 33 +++++++++++++ .../ArithCommon/AttrToLLVMConverter.cpp | 31 ++++++++++++ .../ArithToAMDGPU/ArithToAMDGPU.cpp | 3 ++ .../Conversion/ArithToLLVM/ArithToLLVM.cpp | 32 ++++++++++++- .../Conversion/ArithToSPIRV/ArithToSPIRV.cpp | 9 ++++ mlir/lib/Dialect/Arith/IR/ArithOps.cpp | 46 ++++++++++++++---- .../Dialect/Arith/Transforms/ExpandOps.cpp | 5 ++ .../Conversion/ArithToLLVM/arith-to-llvm.mlir | 15 ++++++ mlir/test/Dialect/Arith/canonicalize.mlir | 45 +++++++++++++++++ mlir/test/Dialect/Arith/ops.mlir | 10 ++++ 13 files changed, 309 insertions(+), 14 deletions(-) diff --git a/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h b/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h index 32d7979c32df..0891e2ba7be7 100644 --- a/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h +++ b/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h @@ -36,6 +36,20 @@ convertArithOverflowFlagsToLLVM(arith::IntegerOverflowFlags arithFlags); LLVM::IntegerOverflowFlagsAttr convertArithOverflowAttrToLLVM(arith::IntegerOverflowFlagsAttr flagsAttr); +/// Creates an LLVM rounding mode enum value from a given arithmetic rounding +/// mode enum value. +LLVM::RoundingMode +convertArithRoundingModeToLLVM(arith::RoundingMode roundingMode); + +/// Creates an LLVM rounding mode attribute from a given arithmetic rounding +/// mode attribute. +LLVM::RoundingModeAttr +convertArithRoundingModeAttrToLLVM(arith::RoundingModeAttr roundingModeAttr); + +/// Returns an attribute for the default LLVM FP exception behavior. +LLVM::FPExceptionBehaviorAttr +getLLVMDefaultFPExceptionBehavior(MLIRContext &context); + // Attribute converter that populates a NamedAttrList by removing the fastmath // attribute from the source operation attributes, and replacing it with an // equivalent LLVM fastmath attribute. @@ -89,6 +103,40 @@ public: private: NamedAttrList convertedAttr; }; + +template +class AttrConverterConstrainedFPToLLVM { + static_assert(TargetOp::template hasTrait< + LLVM::FPExceptionBehaviorOpInterface::Trait>(), + "Target constrained FP operations must implement " + "LLVM::FPExceptionBehaviorOpInterface"); + +public: + AttrConverterConstrainedFPToLLVM(SourceOp srcOp) { + // Copy the source attributes. + convertedAttr = NamedAttrList{srcOp->getAttrs()}; + + if constexpr (TargetOp::template hasTrait< + LLVM::RoundingModeOpInterface::Trait>()) { + // Get the name of the rounding mode attribute. + StringRef arithAttrName = srcOp.getRoundingModeAttrName(); + // Remove the source attribute. + auto arithAttr = + cast(convertedAttr.erase(arithAttrName)); + // Set the target attribute. + convertedAttr.set(TargetOp::getRoundingModeAttrName(), + convertArithRoundingModeAttrToLLVM(arithAttr)); + } + convertedAttr.set(TargetOp::getFPExceptionBehaviorAttrName(), + getLLVMDefaultFPExceptionBehavior(*srcOp->getContext())); + } + + ArrayRef getAttrs() const { return convertedAttr.getAttrs(); } + +private: + NamedAttrList convertedAttr; +}; + } // namespace arith } // namespace mlir diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td b/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td index c8a42c43c880..19a2ade2e95a 100644 --- a/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td +++ b/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td @@ -156,4 +156,29 @@ def Arith_IntegerOverflowAttr : let assemblyFormat = "`<` $value `>`"; } +//===----------------------------------------------------------------------===// +// Arith_RoundingMode +//===----------------------------------------------------------------------===// + +// These correspond to LLVM's values defined in: +// llvm/include/llvm/ADT/FloatingPointMode.h + +def Arith_RToNearestTiesToEven // Round to nearest, ties to even + : I32EnumAttrCase<"to_nearest_even", 0>; +def Arith_RDownward // Round toward -inf + : I32EnumAttrCase<"downward", 1>; +def Arith_RUpward // Round toward +inf + : I32EnumAttrCase<"upward", 2>; +def Arith_RTowardZero // Round toward 0 + : I32EnumAttrCase<"toward_zero", 3>; +def Arith_RToNearestTiesAwayFromZero // Round to nearest, ties away from zero + : I32EnumAttrCase<"to_nearest_away", 4>; + +def Arith_RoundingModeAttr : I32EnumAttr< + "RoundingMode", "Floating point rounding mode", + [Arith_RToNearestTiesToEven, Arith_RDownward, Arith_RUpward, + Arith_RTowardZero, Arith_RToNearestTiesAwayFromZero]> { + let cppNamespace = "::mlir::arith"; +} + #endif // ARITH_BASE diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td index c9df50d0395d..ead19c69a083 100644 --- a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td +++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td @@ -1227,17 +1227,32 @@ def Arith_TruncIOp : Arith_IToICastOp<"trunci"> { // TruncFOp //===----------------------------------------------------------------------===// -def Arith_TruncFOp : Arith_FToFCastOp<"truncf"> { +def Arith_TruncFOp : + Arith_Op<"truncf", + [Pure, SameOperandsAndResultShape, + DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods]>, + Arguments<(ins FloatLike:$in, + OptionalAttr:$roundingmode)>, + Results<(outs FloatLike:$out)> { let summary = "cast from floating-point to narrower floating-point"; let description = [{ Truncate a floating-point value to a smaller floating-point-typed value. The destination type must be strictly narrower than the source type. - If the value cannot be exactly represented, it is rounded using the default - rounding mode. When operating on vectors, casts elementwise. + If the value cannot be exactly represented, it is rounded using the + provided rounding mode or the default one if no rounding mode is provided. + When operating on vectors, casts elementwise. }]; + let builders = [ + OpBuilder<(ins "Type":$out, "Value":$in), [{ + $_state.addOperands(in); + $_state.addTypes(out); + }]> + ]; let hasFolder = 1; let hasVerifier = 1; + let assemblyFormat = "$in ($roundingmode^)? attr-dict `:` type($in) `to` type($out)"; } //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOpsInterfaces.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOpsInterfaces.td index 73a5d9c32ef2..82d6c9ad6b03 100644 --- a/mlir/include/mlir/Dialect/Arith/IR/ArithOpsInterfaces.td +++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOpsInterfaces.td @@ -106,4 +106,37 @@ def ArithIntegerOverflowFlagsInterface : OpInterface<"ArithIntegerOverflowFlagsI ]; } +def ArithRoundingModeInterface : OpInterface<"ArithRoundingModeInterface"> { + let description = [{ + Access to op rounding mode. + }]; + + let cppNamespace = "::mlir::arith"; + + let methods = [ + InterfaceMethod< + /*desc=*/ "Returns a RoundingModeAttr attribute for the operation", + /*returnType=*/ "RoundingModeAttr", + /*methodName=*/ "getRoundingModeAttr", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + auto op = cast(this->getOperation()); + return op.getRoundingmodeAttr(); + }] + >, + StaticInterfaceMethod< + /*desc=*/ [{Returns the name of the RoundingModeAttr attribute for + the operation}], + /*returnType=*/ "StringRef", + /*methodName=*/ "getRoundingModeAttrName", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + return "roundingmode"; + }] + > + ]; +} + #endif // ARITH_OPS_INTERFACES diff --git a/mlir/lib/Conversion/ArithCommon/AttrToLLVMConverter.cpp b/mlir/lib/Conversion/ArithCommon/AttrToLLVMConverter.cpp index dab064a3a954..f12eba98480d 100644 --- a/mlir/lib/Conversion/ArithCommon/AttrToLLVMConverter.cpp +++ b/mlir/lib/Conversion/ArithCommon/AttrToLLVMConverter.cpp @@ -55,3 +55,34 @@ LLVM::IntegerOverflowFlagsAttr mlir::arith::convertArithOverflowAttrToLLVM( return LLVM::IntegerOverflowFlagsAttr::get( flagsAttr.getContext(), convertArithOverflowFlagsToLLVM(arithFlags)); } + +LLVM::RoundingMode +mlir::arith::convertArithRoundingModeToLLVM(arith::RoundingMode roundingMode) { + switch (roundingMode) { + case arith::RoundingMode::downward: + return LLVM::RoundingMode::TowardNegative; + case arith::RoundingMode::to_nearest_away: + return LLVM::RoundingMode::NearestTiesToAway; + case arith::RoundingMode::to_nearest_even: + return LLVM::RoundingMode::NearestTiesToEven; + case arith::RoundingMode::toward_zero: + return LLVM::RoundingMode::TowardZero; + case arith::RoundingMode::upward: + return LLVM::RoundingMode::TowardPositive; + } + llvm_unreachable("Unhandled rounding mode"); +} + +LLVM::RoundingModeAttr mlir::arith::convertArithRoundingModeAttrToLLVM( + arith::RoundingModeAttr roundingModeAttr) { + assert(roundingModeAttr && "Expecting valid attribute"); + return LLVM::RoundingModeAttr::get( + roundingModeAttr.getContext(), + convertArithRoundingModeToLLVM(roundingModeAttr.getValue())); +} + +LLVM::FPExceptionBehaviorAttr +mlir::arith::getLLVMDefaultFPExceptionBehavior(MLIRContext &context) { + return LLVM::FPExceptionBehaviorAttr::get(&context, + LLVM::FPExceptionBehavior::Ignore); +} diff --git a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp index b51a13ae362e..0113a3df0b8e 100644 --- a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp +++ b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp @@ -175,6 +175,9 @@ static Value clampInput(PatternRewriter &rewriter, Location loc, } LogicalResult TruncFToFloat8RewritePattern::match(arith::TruncFOp op) const { + // Only supporting default rounding mode as of now. + if (op.getRoundingmodeAttr()) + return failure(); Type outType = op.getOut().getType(); if (auto outVecType = outType.dyn_cast()) { if (outVecType.isScalable()) diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp index 1f01f4a75c5b..d882f1184f45 100644 --- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp +++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp @@ -28,6 +28,31 @@ using namespace mlir; namespace { +/// Operations whose conversion will depend on whether they are passed a +/// rounding mode attribute or not. +/// +/// `SourceOp` is the source operation; `TargetOp`, the operation it will lower +/// to; `AttrConvert` is the attribute conversion to convert the rounding mode +/// attribute. +template typename AttrConvert = + AttrConvertPassThrough> +struct ConstrainedVectorConvertToLLVMPattern + : public VectorConvertToLLVMPattern { + using VectorConvertToLLVMPattern::VectorConvertToLLVMPattern; + + LogicalResult + matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + if (Constrained != static_cast(op.getRoundingModeAttr())) + return failure(); + return VectorConvertToLLVMPattern::matchAndRewrite(op, adaptor, + rewriter); + } +}; + //===----------------------------------------------------------------------===// // Straightforward Op Lowerings //===----------------------------------------------------------------------===// @@ -112,7 +137,11 @@ using SubIOpLowering = VectorConvertToLLVMPattern; using TruncFOpLowering = - VectorConvertToLLVMPattern; + ConstrainedVectorConvertToLLVMPattern; +using ConstrainedTruncFOpLowering = ConstrainedVectorConvertToLLVMPattern< + arith::TruncFOp, LLVM::ConstrainedFPTruncIntr, true, + arith::AttrConverterConstrainedFPToLLVM>; using TruncIOpLowering = VectorConvertToLLVMPattern; using UIToFPOpLowering = @@ -537,6 +566,7 @@ void mlir::arith::populateArithToLLVMConversionPatterns( SubFOpLowering, SubIOpLowering, TruncFOpLowering, + ConstrainedTruncFOpLowering, TruncIOpLowering, UIToFPOpLowering, XOrIOpLowering diff --git a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp index 7456bf7a87a3..806981728561 100644 --- a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp +++ b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp @@ -805,6 +805,15 @@ struct TypeCastingOpPattern final : public OpConversionPattern { } else { rewriter.template replaceOpWithNewOp(op, dstType, adaptor.getOperands()); + if (auto roundingModeOp = + dyn_cast(*op)) { + if (arith::RoundingModeAttr roundingMode = + roundingModeOp.getRoundingModeAttr()) { + // TODO: Perform rounding mode attribute conversion and attach to new + // operation when defined in the dialect. + return failure(); + } + } } return success(); } diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp index 2f32d9a26e77..0d466795fac0 100644 --- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp +++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp @@ -91,6 +91,29 @@ arith::CmpIPredicate arith::invertPredicate(arith::CmpIPredicate pred) { llvm_unreachable("unknown cmpi predicate kind"); } +/// Equivalent to +/// convertRoundingModeToLLVM(convertArithRoundingModeToLLVM(roundingMode)). +/// +/// Not possible to implement as chain of calls as this would introduce a +/// circular dependency with MLIRArithAttrToLLVMConversion and make arith depend +/// on the LLVM dialect and on translation to LLVM. +static llvm::RoundingMode +convertArithRoundingModeToLLVMIR(RoundingMode roundingMode) { + switch (roundingMode) { + case RoundingMode::downward: + return llvm::RoundingMode::TowardNegative; + case RoundingMode::to_nearest_away: + return llvm::RoundingMode::NearestTiesToAway; + case RoundingMode::to_nearest_even: + return llvm::RoundingMode::NearestTiesToEven; + case RoundingMode::toward_zero: + return llvm::RoundingMode::TowardZero; + case RoundingMode::upward: + return llvm::RoundingMode::TowardPositive; + } + llvm_unreachable("Unhandled rounding mode"); +} + static arith::CmpIPredicateAttr invertPredicate(arith::CmpIPredicateAttr pred) { return arith::CmpIPredicateAttr::get(pred.getContext(), invertPredicate(pred.getValue())); @@ -1233,13 +1256,12 @@ static bool checkWidthChangeCast(TypeRange inputs, TypeRange outputs) { } /// Attempts to convert `sourceValue` to an APFloat value with -/// `targetSemantics`, without any information loss or rounding. -static FailureOr -convertFloatValue(APFloat sourceValue, - const llvm::fltSemantics &targetSemantics) { +/// `targetSemantics` and `roundingMode`, without any information loss. +static FailureOr convertFloatValue( + APFloat sourceValue, const llvm::fltSemantics &targetSemantics, + llvm::RoundingMode roundingMode = llvm::RoundingMode::NearestTiesToEven) { bool losesInfo = false; - auto status = sourceValue.convert( - targetSemantics, llvm::RoundingMode::NearestTiesToEven, &losesInfo); + auto status = sourceValue.convert(targetSemantics, roundingMode, &losesInfo); if (losesInfo || status != APFloat::opOK) return failure(); @@ -1391,15 +1413,19 @@ LogicalResult arith::TruncIOp::verify() { //===----------------------------------------------------------------------===// /// Perform safe const propagation for truncf, i.e., only propagate if FP value -/// can be represented without precision loss or rounding. This is because the -/// semantics of `arith.truncf` do not assume a specific rounding mode. +/// can be represented without precision loss. OpFoldResult arith::TruncFOp::fold(FoldAdaptor adaptor) { auto resElemType = cast(getElementTypeOrSelf(getType())); const llvm::fltSemantics &targetSemantics = resElemType.getFloatSemantics(); return constFoldCastOp( adaptor.getOperands(), getType(), - [&targetSemantics](const APFloat &a, bool &castStatus) { - FailureOr result = convertFloatValue(a, targetSemantics); + [this, &targetSemantics](const APFloat &a, bool &castStatus) { + RoundingMode roundingMode = + getRoundingmode().value_or(RoundingMode::to_nearest_even); + llvm::RoundingMode llvmRoundingMode = + convertArithRoundingModeToLLVMIR(roundingMode); + FailureOr result = + convertFloatValue(a, targetSemantics, llvmRoundingMode); if (failed(result)) { castStatus = false; return a; diff --git a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp index 71e14a153cfd..dd04a5996558 100644 --- a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp +++ b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp @@ -253,6 +253,11 @@ struct BFloat16TruncFOpConverter : public OpRewritePattern { return rewriter.notifyMatchFailure(op, "not a trunc of f32 to bf16."); } + if (op.getRoundingmodeAttr()) { + return rewriter.notifyMatchFailure( + op, "only applicable to default rounding mode."); + } + Type i16Ty = b.getI16Type(); Type i32Ty = b.getI32Type(); Type f32Ty = b.getF32Type(); diff --git a/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir b/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir index 29268eef47e8..56ae930e6d62 100644 --- a/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir +++ b/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir @@ -289,6 +289,21 @@ func.func @fptrunc_vector(%arg0 : vector<2xf32>, %arg1 : vector<2xf64>) { return } +// CHECK-LABEL: experimental_constrained_fptrunc +func.func @experimental_constrained_fptrunc(%arg0 : f64) { +// CHECK-NEXT: = llvm.intr.experimental.constrained.fptrunc {{.*}} tonearest ignore : f64 to f32 + %0 = arith.truncf %arg0 to_nearest_even : f64 to f32 +// CHECK-NEXT: = llvm.intr.experimental.constrained.fptrunc {{.*}} downward ignore : f64 to f32 + %1 = arith.truncf %arg0 downward : f64 to f32 +// CHECK-NEXT: = llvm.intr.experimental.constrained.fptrunc {{.*}} upward ignore : f64 to f32 + %2 = arith.truncf %arg0 upward : f64 to f32 +// CHECK-NEXT: = llvm.intr.experimental.constrained.fptrunc {{.*}} towardzero ignore : f64 to f32 + %3 = arith.truncf %arg0 toward_zero : f64 to f32 +// CHECK-NEXT: = llvm.intr.experimental.constrained.fptrunc {{.*}} tonearestaway ignore : f64 to f32 + %4 = arith.truncf %arg0 to_nearest_away : f64 to f32 + return +} + // Check sign and zero extension and truncation of integers. // CHECK-LABEL: @integer_extension_and_truncation func.func @integer_extension_and_truncation(%arg0 : i3) { diff --git a/mlir/test/Dialect/Arith/canonicalize.mlir b/mlir/test/Dialect/Arith/canonicalize.mlir index bdc6c91d9267..79a318565e98 100644 --- a/mlir/test/Dialect/Arith/canonicalize.mlir +++ b/mlir/test/Dialect/Arith/canonicalize.mlir @@ -757,6 +757,51 @@ func.func @truncFPConstant() -> bf16 { return %0 : bf16 } +// CHECK-LABEL: @truncFPToNearestEvenConstant +// CHECK: %[[cres:.+]] = arith.constant 1.000000e+00 : bf16 +// CHECK: return %[[cres]] +func.func @truncFPToNearestEvenConstant() -> bf16 { + %cst = arith.constant 1.000000e+00 : f32 + %0 = arith.truncf %cst to_nearest_even : f32 to bf16 + return %0 : bf16 +} + +// CHECK-LABEL: @truncFPDownwardConstant +// CHECK: %[[cres:.+]] = arith.constant 1.000000e+00 : bf16 +// CHECK: return %[[cres]] +func.func @truncFPDownwardConstant() -> bf16 { + %cst = arith.constant 1.000000e+00 : f32 + %0 = arith.truncf %cst downward : f32 to bf16 + return %0 : bf16 +} + +// CHECK-LABEL: @truncFPUpwardConstant +// CHECK: %[[cres:.+]] = arith.constant 1.000000e+00 : bf16 +// CHECK: return %[[cres]] +func.func @truncFPUpwardConstant() -> bf16 { + %cst = arith.constant 1.000000e+00 : f32 + %0 = arith.truncf %cst upward : f32 to bf16 + return %0 : bf16 +} + +// CHECK-LABEL: @truncFPTowardZeroConstant +// CHECK: %[[cres:.+]] = arith.constant 1.000000e+00 : bf16 +// CHECK: return %[[cres]] +func.func @truncFPTowardZeroConstant() -> bf16 { + %cst = arith.constant 1.000000e+00 : f32 + %0 = arith.truncf %cst toward_zero : f32 to bf16 + return %0 : bf16 +} + +// CHECK-LABEL: @truncFPToNearestAwayConstant +// CHECK: %[[cres:.+]] = arith.constant 1.000000e+00 : bf16 +// CHECK: return %[[cres]] +func.func @truncFPToNearestAwayConstant() -> bf16 { + %cst = arith.constant 1.000000e+00 : f32 + %0 = arith.truncf %cst to_nearest_away : f32 to bf16 + return %0 : bf16 +} + // CHECK-LABEL: @truncFPVectorConstant // CHECK: %[[cres:.+]] = arith.constant dense<[0.000000e+00, 1.000000e+00]> : vector<2xbf16> // CHECK: return %[[cres]] diff --git a/mlir/test/Dialect/Arith/ops.mlir b/mlir/test/Dialect/Arith/ops.mlir index e499573e324b..f684e02344a5 100644 --- a/mlir/test/Dialect/Arith/ops.mlir +++ b/mlir/test/Dialect/Arith/ops.mlir @@ -703,6 +703,16 @@ func.func @test_truncf_scalable_vector(%arg0 : vector<[8]xf32>) -> vector<[8]xbf return %0 : vector<[8]xbf16> } +// CHECK-LABEL: test_truncf_rounding_mode +func.func @test_truncf_rounding_mode(%arg0 : f64) -> (f32, f32, f32, f32, f32) { + %0 = arith.truncf %arg0 to_nearest_even : f64 to f32 + %1 = arith.truncf %arg0 downward : f64 to f32 + %2 = arith.truncf %arg0 upward : f64 to f32 + %3 = arith.truncf %arg0 toward_zero : f64 to f32 + %4 = arith.truncf %arg0 to_nearest_away : f64 to f32 + return %0, %1, %2, %3, %4 : f32, f32, f32, f32, f32 +} + // CHECK-LABEL: test_uitofp func.func @test_uitofp(%arg0 : i32) -> f32 { %0 = arith.uitofp %arg0 : i32 to f32 -- GitLab From b342d87f89a7cc588abd0d28f69b8dfd9e5cfa0a Mon Sep 17 00:00:00 2001 From: Pengcheng Wang Date: Mon, 1 Apr 2024 18:04:28 +0800 Subject: [PATCH 0104/1690] [TableGen][NFC] Add maybe_unused to MRI (#87044) This suppresses warning `unused variable 'MRI' [-Wunused-variable]` for those fusions that don't need `MRI`. --- llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp index e9e63fa8d0de..63b827a35177 100644 --- a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp +++ b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp @@ -108,7 +108,8 @@ void MacroFusionPredicatorEmitter::emitMacroFusionImpl( OS.indent(4) << "const TargetSubtargetInfo &STI,\n"; OS.indent(4) << "const MachineInstr *FirstMI,\n"; OS.indent(4) << "const MachineInstr &SecondMI) {\n"; - OS.indent(2) << "auto &MRI = SecondMI.getMF()->getRegInfo();\n"; + OS.indent(2) + << "[[maybe_unused]] auto &MRI = SecondMI.getMF()->getRegInfo();\n"; emitPredicates(Predicates, IsCommutable, PE, OS); -- GitLab From a34834138a53f7eb28ba24d325b258ac6f51ae8c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 1 Apr 2024 11:12:21 +0100 Subject: [PATCH 0105/1690] [VPlan] Inline addVPValue into single caller (NFCI). Inline the function into its single caller. --- llvm/lib/Transforms/Vectorize/VPlan.h | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 6c90c793e66c..fdb5c125434a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3006,13 +3006,6 @@ public: void setName(const Twine &newName) { Name = newName.str(); } - void addVPValue(Value *V, VPValue *VPV) { - assert(VPV->isLiveIn() && "VPV must be a live-in."); - assert(V && "Trying to add a null Value to VPlan"); - assert(!Value2VPValue.count(V) && "Value already exists in VPlan"); - Value2VPValue[V] = VPV; - } - /// Gets the live-in VPValue for \p V or adds a new live-in (if none exists /// yet) for \p V. VPValue *getOrAddLiveIn(Value *V) { @@ -3020,7 +3013,9 @@ public: if (!Value2VPValue.count(V)) { VPValue *VPV = new VPValue(V); VPLiveInsToFree.push_back(VPV); - addVPValue(V, VPV); + assert(VPV->isLiveIn() && "VPV must be a live-in."); + assert(!Value2VPValue.count(V) && "Value already exists in VPlan"); + Value2VPValue[V] = VPV; } assert(Value2VPValue.count(V) && "Value does not exist in VPlan"); -- GitLab From c9bcb2b7ddd08e09d75b263273ddb6e0a49a82da Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Mon, 1 Apr 2024 18:16:52 +0800 Subject: [PATCH 0106/1690] [TableGen] Fix MacroFusion.td We are missing `[[maybe_unused]]`. --- llvm/test/TableGen/MacroFusion.td | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/test/TableGen/MacroFusion.td b/llvm/test/TableGen/MacroFusion.td index b54b0506c563..6cf22f544715 100644 --- a/llvm/test/TableGen/MacroFusion.td +++ b/llvm/test/TableGen/MacroFusion.td @@ -83,7 +83,7 @@ def TestSingleFusion: SingleFusion<"test-single-fusion", "HasTestSingleFusion", // CHECK-PREDICATOR-NEXT: const TargetSubtargetInfo &STI, // CHECK-PREDICATOR-NEXT: const MachineInstr *FirstMI, // CHECK-PREDICATOR-NEXT: const MachineInstr &SecondMI) { -// CHECK-PREDICATOR-NEXT: auto &MRI = SecondMI.getMF()->getRegInfo(); +// CHECK-PREDICATOR-NEXT: {{[[]}}{{[[]}}maybe_unused{{[]]}}{{[]]}} auto &MRI = SecondMI.getMF()->getRegInfo(); // CHECK-PREDICATOR-NEXT: { // CHECK-PREDICATOR-NEXT: const MachineInstr *MI = FirstMI; // CHECK-PREDICATOR-NEXT: if (MI->getOperand(0).getReg() != Test::X0) @@ -101,7 +101,7 @@ def TestSingleFusion: SingleFusion<"test-single-fusion", "HasTestSingleFusion", // CHECK-PREDICATOR-NEXT: const TargetSubtargetInfo &STI, // CHECK-PREDICATOR-NEXT: const MachineInstr *FirstMI, // CHECK-PREDICATOR-NEXT: const MachineInstr &SecondMI) { -// CHECK-PREDICATOR-NEXT: auto &MRI = SecondMI.getMF()->getRegInfo(); +// CHECK-PREDICATOR-NEXT: {{[[]}}{{[[]}}maybe_unused{{[]]}}{{[]]}} auto &MRI = SecondMI.getMF()->getRegInfo(); // CHECK-PREDICATOR-NEXT: { // CHECK-PREDICATOR-NEXT: const MachineInstr *MI = &SecondMI; // CHECK-PREDICATOR-NEXT: if (!( @@ -149,7 +149,7 @@ def TestSingleFusion: SingleFusion<"test-single-fusion", "HasTestSingleFusion", // CHECK-PREDICATOR-NEXT: const TargetSubtargetInfo &STI, // CHECK-PREDICATOR-NEXT: const MachineInstr *FirstMI, // CHECK-PREDICATOR-NEXT: const MachineInstr &SecondMI) { -// CHECK-PREDICATOR-NEXT: auto &MRI = SecondMI.getMF()->getRegInfo(); +// CHECK-PREDICATOR-NEXT: {{[[]}}{{[[]}}maybe_unused{{[]]}}{{[]]}} auto &MRI = SecondMI.getMF()->getRegInfo(); // CHECK-PREDICATOR-NEXT: { // CHECK-PREDICATOR-NEXT: const MachineInstr *MI = &SecondMI; // CHECK-PREDICATOR-NEXT: if (!( @@ -185,7 +185,7 @@ def TestSingleFusion: SingleFusion<"test-single-fusion", "HasTestSingleFusion", // CHECK-PREDICATOR-NEXT: const TargetSubtargetInfo &STI, // CHECK-PREDICATOR-NEXT: const MachineInstr *FirstMI, // CHECK-PREDICATOR-NEXT: const MachineInstr &SecondMI) { -// CHECK-PREDICATOR-NEXT: auto &MRI = SecondMI.getMF()->getRegInfo(); +// CHECK-PREDICATOR-NEXT: {{[[]}}{{[[]}}maybe_unused{{[]]}}{{[]]}} auto &MRI = SecondMI.getMF()->getRegInfo(); // CHECK-PREDICATOR-NEXT: { // CHECK-PREDICATOR-NEXT: const MachineInstr *MI = &SecondMI; // CHECK-PREDICATOR-NEXT: if (!( -- GitLab From e701c1a653088488ef67a9fa5b01ab37a482b690 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 1 Apr 2024 12:07:30 +0100 Subject: [PATCH 0107/1690] [VPlan] Use recipe's debug loc for VPWidenMemoryInstructionRecipe (NFCI) Now that VPRecipeBase manages debug locations for recipes, use it in VPWidenMemoryInstructionRecipe. --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 10 +++++----- llvm/lib/Transforms/Vectorize/VPlan.h | 16 +++++++++------- .../lib/Transforms/Vectorize/VPlanTransforms.cpp | 6 ++++-- .../unittests/Transforms/Vectorize/VPlanTest.cpp | 6 +++--- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 452c84f2dcf5..0834865173b2 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8074,11 +8074,11 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef Operands, } if (LoadInst *Load = dyn_cast(I)) return new VPWidenMemoryInstructionRecipe(*Load, Ptr, Mask, Consecutive, - Reverse); + Reverse, I->getDebugLoc()); StoreInst *Store = cast(I); - return new VPWidenMemoryInstructionRecipe(*Store, Ptr, Operands[0], Mask, - Consecutive, Reverse); + return new VPWidenMemoryInstructionRecipe( + *Store, Ptr, Operands[0], Mask, Consecutive, Reverse, I->getDebugLoc()); } /// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also @@ -9340,7 +9340,7 @@ void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) { // Handle Stores: if (SI) { - State.setDebugLocFrom(SI->getDebugLoc()); + State.setDebugLocFrom(getDebugLoc()); for (unsigned Part = 0; Part < State.UF; ++Part) { Instruction *NewSI = nullptr; @@ -9372,7 +9372,7 @@ void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) { // Handle loads. assert(LI && "Must have a load instruction"); - State.setDebugLocFrom(LI->getDebugLoc()); + State.setDebugLocFrom(getDebugLoc()); for (unsigned Part = 0; Part < State.UF; ++Part) { Value *NewLI; if (CreateGatherScatter) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index fdb5c125434a..3baca43f7276 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2278,8 +2278,8 @@ class VPWidenMemoryInstructionRecipe : public VPRecipeBase { public: VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, - bool Consecutive, bool Reverse) - : VPRecipeBase(VPDef::VPWidenMemoryInstructionSC, {Addr}), + bool Consecutive, bool Reverse, DebugLoc DL) + : VPRecipeBase(VPDef::VPWidenMemoryInstructionSC, {Addr}, DL), Ingredient(Load), Consecutive(Consecutive), Reverse(Reverse) { assert((Consecutive || !Reverse) && "Reverse implies consecutive"); new VPValue(this, &Load); @@ -2288,8 +2288,9 @@ public: VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask, - bool Consecutive, bool Reverse) - : VPRecipeBase(VPDef::VPWidenMemoryInstructionSC, {Addr, StoredValue}), + bool Consecutive, bool Reverse, DebugLoc DL) + : VPRecipeBase(VPDef::VPWidenMemoryInstructionSC, {Addr, StoredValue}, + DL), Ingredient(Store), Consecutive(Consecutive), Reverse(Reverse) { assert((Consecutive || !Reverse) && "Reverse implies consecutive"); setMask(Mask); @@ -2299,10 +2300,11 @@ public: if (isStore()) return new VPWidenMemoryInstructionRecipe( cast(Ingredient), getAddr(), getStoredValue(), getMask(), - Consecutive, Reverse); + Consecutive, Reverse, getDebugLoc()); - return new VPWidenMemoryInstructionRecipe( - cast(Ingredient), getAddr(), getMask(), Consecutive, Reverse); + return new VPWidenMemoryInstructionRecipe(cast(Ingredient), + getAddr(), getMask(), Consecutive, + Reverse, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenMemoryInstructionSC) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 6f881d4328f6..957c97cdea5d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -64,11 +64,13 @@ void VPlanTransforms::VPInstructionsToVPRecipes( if (LoadInst *Load = dyn_cast(Inst)) { NewRecipe = new VPWidenMemoryInstructionRecipe( *Load, Ingredient.getOperand(0), nullptr /*Mask*/, - false /*Consecutive*/, false /*Reverse*/); + false /*Consecutive*/, false /*Reverse*/, + Ingredient.getDebugLoc()); } else if (StoreInst *Store = dyn_cast(Inst)) { NewRecipe = new VPWidenMemoryInstructionRecipe( *Store, Ingredient.getOperand(1), Ingredient.getOperand(0), - nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/); + nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/, + Ingredient.getDebugLoc()); } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands()); } else if (CallInst *CI = dyn_cast(Inst)) { diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index e537aac75515..02e7ca341fe2 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -1036,7 +1036,7 @@ TEST(VPRecipeTest, CastVPWidenMemoryInstructionRecipeToVPUserAndVPDef) { new LoadInst(Int32, UndefValue::get(Int32Ptr), "", false, Align(1)); VPValue Addr; VPValue Mask; - VPWidenMemoryInstructionRecipe Recipe(*Load, &Addr, &Mask, true, false); + VPWidenMemoryInstructionRecipe Recipe(*Load, &Addr, &Mask, true, false, {}); EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); @@ -1131,7 +1131,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { new LoadInst(Int32, UndefValue::get(Int32Ptr), "", false, Align(1)); VPValue Addr; VPValue Mask; - VPWidenMemoryInstructionRecipe Recipe(*Load, &Addr, &Mask, true, false); + VPWidenMemoryInstructionRecipe Recipe(*Load, &Addr, &Mask, true, false, {}); EXPECT_FALSE(Recipe.mayHaveSideEffects()); EXPECT_TRUE(Recipe.mayReadFromMemory()); EXPECT_FALSE(Recipe.mayWriteToMemory()); @@ -1146,7 +1146,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { VPValue Mask; VPValue StoredV; VPWidenMemoryInstructionRecipe Recipe(*Store, &Addr, &StoredV, &Mask, false, - false); + false, {}); EXPECT_TRUE(Recipe.mayHaveSideEffects()); EXPECT_FALSE(Recipe.mayReadFromMemory()); EXPECT_TRUE(Recipe.mayWriteToMemory()); -- GitLab From f546b6ef3c15a156959dde16fa5f03a350a0a2be Mon Sep 17 00:00:00 2001 From: harishch4 Date: Mon, 1 Apr 2024 17:13:22 +0530 Subject: [PATCH 0108/1690] =?UTF-8?q?[Flang]=20Relaxing=20an=20error=20whe?= =?UTF-8?q?n=20contiguous=20pointer=20is=20assigned=20to=20a=20non-contig?= =?UTF-8?q?=E2=80=A6=20(#86781)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …uous function. Fix from [thtsikas](https://github.com/thtsikas) based on a discussion in [slack](https://flang-compiler.slack.com/archives/C5C58TT32/p1711124374836079). Example: ``` Program test Integer, Pointer, Contiguous :: cont(:) Interface Function f() Integer, Pointer :: f(:) End Function End Interface cont => f() Print *, cont(3) End Program Function f() Integer, Pointer :: f(:) Allocate (f(4),Source=[1,1,42,1]) ! f => f(4:1:-1) !! not contiguous, runtime error End Function f ``` Understanding is that the standard intended to allow this pattern. The restriction 10.2.2.3 p6 Data pointer assignment "If the pointer object has the CONTIGUOUS attribute, the pointer target shall be contiguous." is not associated with a numbered constraint. If there is a mechanism for injecting runtime checks, this would be a place to do it. Absent that, a warning is the best we can do. No other compiler treats contigPtr => func() as an error when func() is not CONTIGUOUS, so a warning would probably be better for consistency. https://godbolt.org/z/5cM6roeEE --- flang/lib/Semantics/pointer-assignment.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flang/lib/Semantics/pointer-assignment.cpp b/flang/lib/Semantics/pointer-assignment.cpp index 58155a29da1e..4b4ce153084d 100644 --- a/flang/lib/Semantics/pointer-assignment.cpp +++ b/flang/lib/Semantics/pointer-assignment.cpp @@ -266,7 +266,7 @@ bool PointerAssignmentChecker::Check(const evaluate::FunctionRef &f) { } else if (isContiguous_ && !funcResult->attrs.test(FunctionResult::Attr::Contiguous)) { msg = "CONTIGUOUS %s is associated with the result of reference to" - " function '%s' that is not contiguous"_err_en_US; + " function '%s' that is not known to be contiguous"_warn_en_US; } else if (lhsType_) { const auto *frTypeAndShape{funcResult->GetTypeAndShape()}; CHECK(frTypeAndShape); -- GitLab From 4213f4a9ae0ef70e02da9f40653b4e04eea00c74 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 1 Apr 2024 07:28:34 -0500 Subject: [PATCH 0109/1690] [Libomptarget] Fix resizing the buffer of RPC handles Summary: The previous code would potentially make it smaller if a device with a lower ID touched it later. Also we should minimize changes to the state for multi threaded reasons. This just sets up an owned slot for each at initialization time. --- openmp/libomptarget/plugins-nextgen/common/include/RPC.h | 4 ++++ .../plugins-nextgen/common/src/PluginInterface.cpp | 2 +- openmp/libomptarget/plugins-nextgen/common/src/RPC.cpp | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/common/include/RPC.h b/openmp/libomptarget/plugins-nextgen/common/include/RPC.h index b621cc0da458..01bf539bcb3f 100644 --- a/openmp/libomptarget/plugins-nextgen/common/include/RPC.h +++ b/openmp/libomptarget/plugins-nextgen/common/include/RPC.h @@ -23,6 +23,7 @@ namespace llvm::omp::target { namespace plugin { +struct GenericPluginTy; struct GenericDeviceTy; class GenericGlobalHandlerTy; class DeviceImageTy; @@ -33,6 +34,9 @@ class DeviceImageTy; /// these routines will perform no action. struct RPCServerTy { public: + /// Initializes the handles to the number of devices we may need to service. + RPCServerTy(plugin::GenericPluginTy &Plugin); + /// Check if this device image is using an RPC server. This checks for the /// precense of an externally visible symbol in the device image that will /// be present whenever RPC code is called. diff --git a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp index 55e2865d6aae..b5f3c45c835f 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp @@ -1492,7 +1492,7 @@ Error GenericPluginTy::init() { GlobalHandler = createGlobalHandler(); assert(GlobalHandler && "Invalid global handler"); - RPCServer = new RPCServerTy(); + RPCServer = new RPCServerTy(*this); assert(RPCServer && "Invalid RPC server"); return Plugin::success(); diff --git a/openmp/libomptarget/plugins-nextgen/common/src/RPC.cpp b/openmp/libomptarget/plugins-nextgen/common/src/RPC.cpp index fab0f6838f4a..faa2cbd4f02f 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/RPC.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/RPC.cpp @@ -21,6 +21,9 @@ using namespace llvm; using namespace omp; using namespace target; +RPCServerTy::RPCServerTy(plugin::GenericPluginTy &Plugin) + : Handles(Plugin.getNumDevices()) {} + llvm::Expected RPCServerTy::isDeviceUsingRPC(plugin::GenericDeviceTy &Device, plugin::GenericGlobalHandlerTy &Handler, @@ -101,7 +104,6 @@ Error RPCServerTy::initDevice(plugin::GenericDeviceTy &Device, if (auto Err = Device.dataSubmit(ClientPtr, ClientBuffer, rpc_get_client_size(), nullptr)) return Err; - Handles.resize(Device.getDeviceId() + 1); Handles[Device.getDeviceId()] = RPCDevice.handle; #endif return Error::success(); -- GitLab From da9f06c9b1179423302e3e7ccb27431ced44e548 Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Mon, 1 Apr 2024 08:46:22 -0400 Subject: [PATCH 0110/1690] [GISEL] G_SPLAT_VECTOR can take a splat that is larger than the vector element (#86974) This is what SelectionDAG does. We'd like to reuse SelectionDAG patterns. --- llvm/docs/GlobalISel/GenericOpcode.rst | 4 ++++ llvm/lib/CodeGen/MachineVerifier.cpp | 17 ++++++++++++----- .../MachineVerifier/test_g_splat_vector.mir | 4 ++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst index cae2c21b80d7..a12627c01d20 100644 --- a/llvm/docs/GlobalISel/GenericOpcode.rst +++ b/llvm/docs/GlobalISel/GenericOpcode.rst @@ -690,6 +690,10 @@ G_SPLAT_VECTOR Create a vector where all elements are the scalar from the source operand. +The type of the operand must be equal to or larger than the vector element +type. If the operand is larger than the vector element type, the scalar is +implicitly truncated to the vector element type. + Vector Reduction Operations --------------------------- diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index e4e05ce9278c..fd7ea2842647 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1768,16 +1768,23 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); - if (!DstTy.isScalableVector()) + if (!DstTy.isScalableVector()) { report("Destination type must be a scalable vector", MI); + break; + } - if (!SrcTy.isScalar()) + if (!SrcTy.isScalar()) { report("Source type must be a scalar", MI); + break; + } - if (DstTy.getScalarType() != SrcTy) - report("Element type of the destination must be the same type as the " - "source type", + if (TypeSize::isKnownGT(DstTy.getElementType().getSizeInBits(), + SrcTy.getSizeInBits())) { + report("Element type of the destination must be the same size or smaller " + "than the source type", MI); + break; + } break; } diff --git a/llvm/test/MachineVerifier/test_g_splat_vector.mir b/llvm/test/MachineVerifier/test_g_splat_vector.mir index 0d1d8a3e6dcc..00074349776f 100644 --- a/llvm/test/MachineVerifier/test_g_splat_vector.mir +++ b/llvm/test/MachineVerifier/test_g_splat_vector.mir @@ -22,6 +22,6 @@ body: | ; CHECK: Source type must be a scalar %6:_() = G_SPLAT_VECTOR %2 - ; CHECK: Element type of the destination must be the same type as the source type - %7:_() = G_SPLAT_VECTOR %0 + ; CHECK: Element type of the destination must be the same size or smaller than the source type + %7:_() = G_SPLAT_VECTOR %0 ... -- GitLab From 41afef9066eec8daf517ac357a628cdf30c95e39 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Mon, 1 Apr 2024 04:20:08 -0700 Subject: [PATCH 0111/1690] [SLP]Fix PR87011: Missing sign extension of demoted type before zero extension Need to drop skipping of the first zext/sext nodes, it leads to incorrect and less profitable code. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 6 ++--- .../RISCV/init-ext-node-not-truncable.ll | 2 +- .../SLPVectorizer/X86/sext-inseltpoison.ll | 22 +++++-------------- .../test/Transforms/SLPVectorizer/X86/sext.ll | 22 +++++-------------- .../SLPVectorizer/X86/zext-inseltpoison.ll | 9 ++------ .../test/Transforms/SLPVectorizer/X86/zext.ll | 9 ++------ 6 files changed, 19 insertions(+), 51 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 2bc0c5dcc606..1ffc39a90674 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -14263,11 +14263,9 @@ void BoUpSLP::computeMinimumValueSizes() { SmallVector RootDemotes; if (NodeIdx != 0 && VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize && - (VectorizableTree[NodeIdx]->getOpcode() == Instruction::ZExt || - VectorizableTree[NodeIdx]->getOpcode() == Instruction::SExt || - VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc)) { + VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) { assert(IsStoreOrInsertElt && "Expected store/insertelement seeded graph."); - IsTruncRoot = VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc; + IsTruncRoot = true; RootDemotes.push_back(NodeIdx); IsProfitableToDemoteRoot = true; ++NodeIdx; diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/init-ext-node-not-truncable.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/init-ext-node-not-truncable.ll index 436fba3261d6..1166b1fca826 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/init-ext-node-not-truncable.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/init-ext-node-not-truncable.ll @@ -7,7 +7,7 @@ define void @test() { ; CHECK-LABEL: define void @test( ; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: entry: -; CHECK-NEXT: store <2 x i64> , ptr @h, align 8 +; CHECK-NEXT: store <2 x i64> , ptr @h, align 8 ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sext-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/sext-inseltpoison.ll index 5ae0ad932fdd..b64743aaa283 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/sext-inseltpoison.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/sext-inseltpoison.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE,SSE2 -; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE,SLM +; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX @@ -11,20 +11,10 @@ ; define <2 x i64> @loadext_2i8_to_2i64(ptr %p0) { -; SSE2-LABEL: @loadext_2i8_to_2i64( -; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P0:%.*]], i64 1 -; SSE2-NEXT: [[I0:%.*]] = load i8, ptr [[P0]], align 1 -; SSE2-NEXT: [[I1:%.*]] = load i8, ptr [[P1]], align 1 -; SSE2-NEXT: [[X0:%.*]] = sext i8 [[I0]] to i64 -; SSE2-NEXT: [[X1:%.*]] = sext i8 [[I1]] to i64 -; SSE2-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[X0]], i32 0 -; SSE2-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1 -; SSE2-NEXT: ret <2 x i64> [[V1]] -; -; SLM-LABEL: @loadext_2i8_to_2i64( -; SLM-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[P0:%.*]], align 1 -; SLM-NEXT: [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i64> -; SLM-NEXT: ret <2 x i64> [[TMP3]] +; SSE-LABEL: @loadext_2i8_to_2i64( +; SSE-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr [[P0:%.*]], align 1 +; SSE-NEXT: [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i64> +; SSE-NEXT: ret <2 x i64> [[TMP2]] ; ; AVX-LABEL: @loadext_2i8_to_2i64( ; AVX-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[P0:%.*]], align 1 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sext.ll b/llvm/test/Transforms/SLPVectorizer/X86/sext.ll index 7d38aeb0c363..744a50906cfc 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/sext.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/sext.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE,SSE2 -; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE,SLM +; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX @@ -11,20 +11,10 @@ ; define <2 x i64> @loadext_2i8_to_2i64(ptr %p0) { -; SSE2-LABEL: @loadext_2i8_to_2i64( -; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P0:%.*]], i64 1 -; SSE2-NEXT: [[I0:%.*]] = load i8, ptr [[P0]], align 1 -; SSE2-NEXT: [[I1:%.*]] = load i8, ptr [[P1]], align 1 -; SSE2-NEXT: [[X0:%.*]] = sext i8 [[I0]] to i64 -; SSE2-NEXT: [[X1:%.*]] = sext i8 [[I1]] to i64 -; SSE2-NEXT: [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0 -; SSE2-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1 -; SSE2-NEXT: ret <2 x i64> [[V1]] -; -; SLM-LABEL: @loadext_2i8_to_2i64( -; SLM-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[P0:%.*]], align 1 -; SLM-NEXT: [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i64> -; SLM-NEXT: ret <2 x i64> [[TMP3]] +; SSE-LABEL: @loadext_2i8_to_2i64( +; SSE-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr [[P0:%.*]], align 1 +; SSE-NEXT: [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i64> +; SSE-NEXT: ret <2 x i64> [[TMP2]] ; ; AVX-LABEL: @loadext_2i8_to_2i64( ; AVX-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[P0:%.*]], align 1 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/zext-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/zext-inseltpoison.ll index d1f6c41e5c30..27996a7064c0 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/zext-inseltpoison.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/zext-inseltpoison.ll @@ -12,13 +12,8 @@ define <2 x i64> @loadext_2i8_to_2i64(ptr %p0) { ; SSE2-LABEL: @loadext_2i8_to_2i64( -; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P0:%.*]], i64 1 -; SSE2-NEXT: [[I0:%.*]] = load i8, ptr [[P0]], align 1 -; SSE2-NEXT: [[I1:%.*]] = load i8, ptr [[P1]], align 1 -; SSE2-NEXT: [[X0:%.*]] = zext i8 [[I0]] to i64 -; SSE2-NEXT: [[X1:%.*]] = zext i8 [[I1]] to i64 -; SSE2-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[X0]], i32 0 -; SSE2-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1 +; SSE2-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr [[P0:%.*]], align 1 +; SSE2-NEXT: [[V1:%.*]] = zext <2 x i8> [[TMP1]] to <2 x i64> ; SSE2-NEXT: ret <2 x i64> [[V1]] ; ; SLM-LABEL: @loadext_2i8_to_2i64( diff --git a/llvm/test/Transforms/SLPVectorizer/X86/zext.ll b/llvm/test/Transforms/SLPVectorizer/X86/zext.ll index 829e4bab20ff..94870420f2bf 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/zext.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/zext.ll @@ -12,13 +12,8 @@ define <2 x i64> @loadext_2i8_to_2i64(ptr %p0) { ; SSE2-LABEL: @loadext_2i8_to_2i64( -; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P0:%.*]], i64 1 -; SSE2-NEXT: [[I0:%.*]] = load i8, ptr [[P0]], align 1 -; SSE2-NEXT: [[I1:%.*]] = load i8, ptr [[P1]], align 1 -; SSE2-NEXT: [[X0:%.*]] = zext i8 [[I0]] to i64 -; SSE2-NEXT: [[X1:%.*]] = zext i8 [[I1]] to i64 -; SSE2-NEXT: [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0 -; SSE2-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1 +; SSE2-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr [[P0:%.*]], align 1 +; SSE2-NEXT: [[V1:%.*]] = zext <2 x i8> [[TMP1]] to <2 x i64> ; SSE2-NEXT: ret <2 x i64> [[V1]] ; ; SLM-LABEL: @loadext_2i8_to_2i64( -- GitLab From a7206a6fa32ada15578e3afddcc1480364c25f4c Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Mon, 1 Apr 2024 09:35:42 -0400 Subject: [PATCH 0112/1690] [RISCV] ReadStoreData is read later in the pipeline for SiFive7 (#86454) Store data is read later in the pipeline, so we use SiFive7AnyToGPRBypass to model that a store instruction can begin some cycles before that data is ready. --- llvm/lib/Target/RISCV/RISCVSchedSiFive7.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td index 3586d235bdbb..54a13889ee69 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td +++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td @@ -950,7 +950,7 @@ def : InstRW<[WriteIALU], (instrs COPY)>; def : SiFive7AnyToGPRBypass; def : SiFive7AnyToGPRBypass; def : ReadAdvance; -def : ReadAdvance; +def : SiFive7AnyToGPRBypass; def : ReadAdvance; def : SiFive7AnyToGPRBypass; def : SiFive7AnyToGPRBypass; -- GitLab From 971b852546a7d96bc8887ced913724b884cf40df Mon Sep 17 00:00:00 2001 From: Jakub Kuderski Date: Mon, 1 Apr 2024 11:40:09 -0400 Subject: [PATCH 0113/1690] [mlir][NFC] Simplify type checks with isa predicates (#87183) For more context on isa predicates, see: https://github.com/llvm/llvm-project/pull/83753. --- .../Conversion/GPUCommon/GPUOpsLowering.cpp | 3 +- .../Conversion/VectorToGPU/VectorToGPU.cpp | 10 ++--- .../Affine/Analysis/AffineAnalysis.cpp | 3 +- .../Affine/Transforms/SuperVectorize.cpp | 5 +-- .../Dialect/Affine/Utils/LoopFusionUtils.cpp | 3 +- .../FuncBufferizableOpInterfaceImpl.cpp | 2 +- .../Transforms/OneShotModuleBufferize.cpp | 8 ++-- mlir/lib/Dialect/EmitC/IR/EmitC.cpp | 3 +- .../GPU/TransformOps/GPUTransformOps.cpp | 24 ++++------- .../GPU/Transforms/AsyncRegionRewriter.cpp | 5 +-- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 6 +-- mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp | 6 +-- .../Linalg/Transforms/ElementwiseToLinalg.cpp | 3 +- .../Linalg/Transforms/Vectorization.cpp | 9 ++-- mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp | 2 +- .../Dialect/SPIRV/IR/CooperativeMatrixOps.cpp | 3 +- mlir/lib/Dialect/Shape/IR/Shape.cpp | 5 +-- mlir/lib/Dialect/Traits.cpp | 15 ++++--- .../lib/Dialect/Transform/IR/TransformOps.cpp | 2 +- mlir/lib/Dialect/Vector/IR/VectorOps.cpp | 9 ++-- .../Vector/Transforms/VectorDistribute.cpp | 41 +++++++++---------- mlir/lib/IR/AffineMap.cpp | 4 +- mlir/lib/IR/Operation.cpp | 4 +- mlir/lib/TableGen/Class.cpp | 4 +- mlir/lib/Target/Cpp/TranslateToCpp.cpp | 5 +-- .../Target/SPIRV/Serialization/Serializer.cpp | 8 ++-- .../Transforms/Utils/DialectConversion.cpp | 6 +-- .../Transforms/Utils/OneToNTypeConversion.cpp | 3 +- 28 files changed, 83 insertions(+), 118 deletions(-) diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp index 73d418cb8413..993c09b03c0f 100644 --- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp +++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp @@ -545,8 +545,7 @@ LogicalResult impl::scalarizeVectorOp(Operation *op, ValueRange operands, ConversionPatternRewriter &rewriter, const LLVMTypeConverter &converter) { TypeRange operandTypes(operands); - if (llvm::none_of(operandTypes, - [](Type type) { return isa(type); })) { + if (llvm::none_of(operandTypes, llvm::IsaPred)) { return rewriter.notifyMatchFailure(op, "expected vector operand"); } if (op->getNumRegions() != 0 || op->getNumSuccessors() != 0) diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp index 85fb8a539912..399c0450824e 100644 --- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp +++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp @@ -202,9 +202,7 @@ template static bool integerExtendSupportsMMAMatrixType(ExtOpTy extOp) { if (!isa(extOp.getOperand().getDefiningOp())) return false; - return llvm::all_of(extOp->getUsers(), [](Operation *user) { - return isa(user); - }); + return llvm::all_of(extOp->getUsers(), llvm::IsaPred); } static bool fpExtendSupportsMMAMatrixType(arith::ExtFOp extOp) { return true; } @@ -345,15 +343,13 @@ getSliceContract(Operation *op, static SetVector getOpToConvert(mlir::Operation *op, bool useNvGpu) { auto hasVectorDest = [](Operation *op) { - return llvm::any_of(op->getResultTypes(), - [](Type t) { return isa(t); }); + return llvm::any_of(op->getResultTypes(), llvm::IsaPred); }; BackwardSliceOptions backwardSliceOptions; backwardSliceOptions.filter = hasVectorDest; auto hasVectorSrc = [](Operation *op) { - return llvm::any_of(op->getOperandTypes(), - [](Type t) { return isa(t); }); + return llvm::any_of(op->getOperandTypes(), llvm::IsaPred); }; ForwardSliceOptions forwardSliceOptions; forwardSliceOptions.filter = hasVectorSrc; diff --git a/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp b/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp index 61244921bc38..69b3d41e17c2 100644 --- a/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp +++ b/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp @@ -136,8 +136,7 @@ static bool isLocallyDefined(Value v, Operation *enclosingOp) { bool mlir::affine::isLoopMemoryParallel(AffineForOp forOp) { // Any memref-typed iteration arguments are treated as serializing. - if (llvm::any_of(forOp.getResultTypes(), - [](Type type) { return isa(type); })) + if (llvm::any_of(forOp.getResultTypes(), llvm::IsaPred)) return false; // Collect all load and store ops in loop nest rooted at 'forOp'. diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp index 46c7871f4023..71e9648a5e00 100644 --- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp +++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp @@ -609,9 +609,8 @@ makePattern(const DenseSet ¶llelLoops, int vectorRank, } static NestedPattern &vectorTransferPattern() { - static auto pattern = affine::matcher::Op([](Operation &op) { - return isa(op); - }); + static auto pattern = affine::matcher::Op( + llvm::IsaPred); return pattern; } diff --git a/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp index fb45528ad5e7..84ae4b52dcf4 100644 --- a/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp +++ b/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp @@ -211,8 +211,7 @@ static unsigned getMaxLoopDepth(ArrayRef srcOps, unsigned loopDepth = getInnermostCommonLoopDepth(targetDstOps); // Return common loop depth for loads if there are no store ops. - if (all_of(targetDstOps, - [&](Operation *op) { return isa(op); })) + if (all_of(targetDstOps, llvm::IsaPred)) return loopDepth; // Check dependences on all pairs of ops in 'targetDstOps' and store the diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp index 4cdbbf35dc87..053ea7935260 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp @@ -326,7 +326,7 @@ struct FuncOpInterface static bool supportsUnstructuredControlFlow() { return true; } bool hasTensorSemantics(Operation *op) const { - auto isaTensor = [](Type type) { return isa(type); }; + auto isaTensor = llvm::IsaPred; // A function has tensor semantics if it has tensor arguments/results. auto funcOp = cast(op); diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OneShotModuleBufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OneShotModuleBufferize.cpp index 33feea0b956c..0a4072605c26 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/OneShotModuleBufferize.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/OneShotModuleBufferize.cpp @@ -67,6 +67,7 @@ #include "mlir/Dialect/Bufferization/Transforms/Transforms.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/Operation.h" using namespace mlir; @@ -277,9 +278,10 @@ static void equivalenceAnalysis(func::FuncOp funcOp, /// Return "true" if the given function signature has tensor semantics. static bool hasTensorSignature(func::FuncOp funcOp) { - auto isaTensor = [](Type t) { return isa(t); }; - return llvm::any_of(funcOp.getFunctionType().getInputs(), isaTensor) || - llvm::any_of(funcOp.getFunctionType().getResults(), isaTensor); + return llvm::any_of(funcOp.getFunctionType().getInputs(), + llvm::IsaPred) || + llvm::any_of(funcOp.getFunctionType().getResults(), + llvm::IsaPred); } /// Store all functions of the `moduleOp` in `orderedFuncOps`, sorted by diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp index ab5c418e844f..f4a9dc3ca509 100644 --- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp +++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp @@ -224,8 +224,7 @@ LogicalResult emitc::CallOpaqueOp::verify() { } } - if (llvm::any_of(getResultTypes(), - [](Type type) { return isa(type); })) { + if (llvm::any_of(getResultTypes(), llvm::IsaPred)) { return emitOpError() << "cannot return array type"; } diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp index fc3a43756945..b584f63f16e0 100644 --- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp +++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp @@ -296,22 +296,14 @@ checkMappingAttributeTypes(std::optional transformOp, "scf.forall op requires a mapping attribute"); } - bool hasBlockMapping = - llvm::any_of(forallOp.getMapping().value(), [](Attribute attr) { - return isa(attr); - }); - bool hasWarpgroupMapping = - llvm::any_of(forallOp.getMapping().value(), [](Attribute attr) { - return isa(attr); - }); - bool hasWarpMapping = - llvm::any_of(forallOp.getMapping().value(), [](Attribute attr) { - return isa(attr); - }); - bool hasThreadMapping = - llvm::any_of(forallOp.getMapping().value(), [](Attribute attr) { - return isa(attr); - }); + bool hasBlockMapping = llvm::any_of(forallOp.getMapping().value(), + llvm::IsaPred); + bool hasWarpgroupMapping = llvm::any_of( + forallOp.getMapping().value(), llvm::IsaPred); + bool hasWarpMapping = llvm::any_of(forallOp.getMapping().value(), + llvm::IsaPred); + bool hasThreadMapping = llvm::any_of(forallOp.getMapping().value(), + llvm::IsaPred); int64_t countMappingTypes = 0; countMappingTypes += hasBlockMapping ? 1 : 0; countMappingTypes += hasWarpgroupMapping ? 1 : 0; diff --git a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp index 40903f199afd..b2fa3a99c53f 100644 --- a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp +++ b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp @@ -232,9 +232,8 @@ private: // control flow code. static bool areAllUsersExecuteOrAwait(Value token) { return !token.use_empty() && - llvm::all_of(token.getUsers(), [](Operation *user) { - return isa(user); - }); + llvm::all_of(token.getUsers(), + llvm::IsaPred); } // Add the `asyncToken` as dependency as needed after `op`. diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 3ba6ac6ccc81..e5c19a916392 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2786,10 +2786,8 @@ LogicalResult LLVM::BitcastOp::verify() { if (!resultType) return success(); - auto isVector = [](Type type) { - return llvm::isa( - type); - }; + auto isVector = + llvm::IsaPred; // Due to bitcast requiring both operands to be of the same size, it is not // possible for only one of the two to be a pointer of vectors. diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp index 6954eee93efd..2d7219fef87c 100644 --- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp +++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp @@ -28,6 +28,7 @@ #include "mlir/IR/AffineExprVisitor.h" #include "mlir/IR/AffineMap.h" #include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/BuiltinTypeInterfaces.h" #include "mlir/IR/Matchers.h" #include "mlir/IR/OpImplementation.h" #include "mlir/IR/OperationSupport.h" @@ -119,8 +120,7 @@ static void fillStructuredOpRegion(OpBuilder &opBuilder, Region ®ion, TypeRange inputTypes, TypeRange outputTypes, ArrayRef attrs, RegionBuilderFn regionBuilder) { - assert(llvm::all_of(outputTypes, - [](Type t) { return llvm::isa(t); })); + assert(llvm::all_of(outputTypes, llvm::IsaPred)); SmallVector argTypes; SmallVector argLocs; @@ -162,7 +162,7 @@ static void buildStructuredOp(OpBuilder &b, OperationState &state, resultTensorTypes.value_or(TypeRange()); if (!resultTensorTypes) copy_if(outputs.getTypes(), std::back_inserter(derivedResultTypes), - [](Type type) { return llvm::isa(type); }); + llvm::IsaPred); state.addOperands(inputs); state.addOperands(outputs); diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp index 5508aaf9d875..28d6752fc2d3 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp @@ -27,8 +27,7 @@ static bool isElementwiseMappableOpOnRankedTensors(Operation *op) { // TODO: The conversion pattern can be made to work for `any_of` here, but // it's more complex as it requires tracking which operands are scalars. - return llvm::all_of(op->getOperandTypes(), - [](Type type) { return isa(type); }); + return llvm::all_of(op->getOperandTypes(), llvm::IsaPred); } /// Given `op` assumed `isElementwiseMappableOpOnRankedTensors`, iterate over diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp index c74ab1e6448b..25785653a716 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp @@ -3537,15 +3537,14 @@ private: // Otherwise, check for one or zero `ext` predecessor. The `ext` operands // must be block arguments or extension of block arguments. bool setOperKind(Operation *reduceOp) { - int numBlockArguments = llvm::count_if( - reduceOp->getOperands(), [](Value v) { return isa(v); }); + int numBlockArguments = + llvm::count_if(reduceOp->getOperands(), llvm::IsaPred); switch (numBlockArguments) { case 1: { // Will be convolution if feeder is a MulOp. // Otherwise, if it can be pooling. - auto feedValIt = llvm::find_if(reduceOp->getOperands(), [](Value v) { - return !isa(v); - }); + auto feedValIt = llvm::find_if_not(reduceOp->getOperands(), + llvm::IsaPred); Operation *feedOp = (*feedValIt).getDefiningOp(); if (isCastOfBlockArgument(feedOp)) { oper = Pool; diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp index c09a3403f9a3..9ba96e4be7d1 100644 --- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp +++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp @@ -457,7 +457,7 @@ static ParseResult parseRegions(OpAsmParser &parser, OperationState &state, } static bool isComputeOperation(Operation *op) { - return isa(op) || isa(op); + return isa(op); } namespace { diff --git a/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp b/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp index d532d466334a..2ff3efdc96a7 100644 --- a/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp +++ b/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp @@ -125,8 +125,7 @@ LogicalResult KHRCooperativeMatrixMulAddOp::verify() { if (getMatrixOperands()) { Type elementTypes[] = {typeA.getElementType(), typeB.getElementType(), typeC.getElementType()}; - if (!llvm::all_of(elementTypes, - [](Type ty) { return isa(ty); })) { + if (!llvm::all_of(elementTypes, llvm::IsaPred)) { return emitOpError("Matrix Operands require all matrix element types to " "be Integer Types"); } diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp index f5a3717f815d..58c3f4c33457 100644 --- a/mlir/lib/Dialect/Shape/IR/Shape.cpp +++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp @@ -65,9 +65,8 @@ LogicalResult shape::getShapeVec(Value input, } static bool isErrorPropagationPossible(TypeRange operandTypes) { - return llvm::any_of(operandTypes, [](Type ty) { - return llvm::isa(ty); - }); + return llvm::any_of(operandTypes, + llvm::IsaPred); } static LogicalResult verifySizeOrIndexOp(Operation *op) { diff --git a/mlir/lib/Dialect/Traits.cpp b/mlir/lib/Dialect/Traits.cpp index d4e0f8a31370..2efc157ce796 100644 --- a/mlir/lib/Dialect/Traits.cpp +++ b/mlir/lib/Dialect/Traits.cpp @@ -188,9 +188,8 @@ Type OpTrait::util::getBroadcastedType(Type type1, Type type2, /// Returns a tuple corresponding to whether range has tensor or vector type. template static std::tuple hasTensorOrVectorType(iterator_range types) { - return std::make_tuple( - llvm::any_of(types, [](Type t) { return isa(t); }), - llvm::any_of(types, [](Type t) { return isa(t); })); + return {llvm::any_of(types, llvm::IsaPred), + llvm::any_of(types, llvm::IsaPred)}; } static bool isCompatibleInferredReturnShape(ArrayRef inferred, @@ -202,7 +201,7 @@ static bool isCompatibleInferredReturnShape(ArrayRef inferred, }; if (inferred.size() != existing.size()) return false; - for (auto [inferredDim, existingDim] : llvm::zip(inferred, existing)) + for (auto [inferredDim, existingDim] : llvm::zip_equal(inferred, existing)) if (!isCompatible(inferredDim, existingDim)) return false; return true; @@ -238,8 +237,8 @@ LogicalResult OpTrait::impl::verifyCompatibleOperandBroadcast(Operation *op) { std::get<1>(resultsHasTensorVectorType))) return op->emitError("cannot broadcast vector with tensor"); - auto rankedOperands = make_filter_range( - op->getOperandTypes(), [](Type t) { return isa(t); }); + auto rankedOperands = + make_filter_range(op->getOperandTypes(), llvm::IsaPred); // If all operands are unranked, then all result shapes are possible. if (rankedOperands.empty()) @@ -257,8 +256,8 @@ LogicalResult OpTrait::impl::verifyCompatibleOperandBroadcast(Operation *op) { return op->emitOpError("operands don't have broadcast-compatible shapes"); } - auto rankedResults = make_filter_range( - op->getResultTypes(), [](Type t) { return isa(t); }); + auto rankedResults = + make_filter_range(op->getResultTypes(), llvm::IsaPred); // If all of the results are unranked then no further verification. if (rankedResults.empty()) diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp index 578b2492bbab..c8d06ba157b9 100644 --- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp +++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp @@ -819,7 +819,7 @@ bool transform::CastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) { assert(outputs.size() == 1 && "expected one output"); return llvm::all_of( std::initializer_list{inputs.front(), outputs.front()}, - [](Type ty) { return isa(ty); }); + llvm::IsaPred); } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp index e566bfacf379..3e6425879cc6 100644 --- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp +++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp @@ -898,13 +898,12 @@ static LogicalResult verifyOutputShape( AffineMap resMap = op.getIndexingMapsArray()[2]; auto extentsMap = AffineMap::get(/*dimCount=*/extents.size(), - /*symCount=*/0, extents, ctx); + /*symbolCount=*/0, extents, ctx); // Compose the resMap with the extentsMap, which is a constant map. AffineMap expectedMap = simplifyAffineMap(resMap.compose(extentsMap)); - assert( - llvm::all_of(expectedMap.getResults(), - [](AffineExpr e) { return isa(e); }) && - "expected constant extent along all dimensions."); + assert(llvm::all_of(expectedMap.getResults(), + llvm::IsaPred) && + "expected constant extent along all dimensions."); // Extract the expected shape and build the type. auto expectedShape = llvm::to_vector<4>( llvm::map_range(expectedMap.getResults(), [](AffineExpr e) { diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp index b3ab4a916121..a67e03e85f71 100644 --- a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp @@ -598,9 +598,8 @@ struct WarpOpTransferWrite : public OpRewritePattern { } // Do not process warp ops that contain only TransferWriteOps. - if (llvm::all_of(warpOp.getOps(), [](Operation &op) { - return isa(&op); - })) + if (llvm::all_of(warpOp.getOps(), + llvm::IsaPred)) return failure(); SmallVector yieldValues = {writeOp.getVector()}; @@ -746,8 +745,8 @@ struct WarpOpConstant : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp, PatternRewriter &rewriter) const override { - OpOperand *yieldOperand = getWarpResult( - warpOp, [](Operation *op) { return isa(op); }); + OpOperand *yieldOperand = + getWarpResult(warpOp, llvm::IsaPred); if (!yieldOperand) return failure(); auto constantOp = yieldOperand->get().getDefiningOp(); @@ -1060,8 +1059,8 @@ struct WarpOpBroadcast : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp, PatternRewriter &rewriter) const override { - OpOperand *operand = getWarpResult( - warpOp, [](Operation *op) { return isa(op); }); + OpOperand *operand = + getWarpResult(warpOp, llvm::IsaPred); if (!operand) return failure(); unsigned int operandNumber = operand->getOperandNumber(); @@ -1097,8 +1096,8 @@ struct WarpOpShapeCast : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp, PatternRewriter &rewriter) const override { - OpOperand *operand = getWarpResult( - warpOp, [](Operation *op) { return isa(op); }); + OpOperand *operand = + getWarpResult(warpOp, llvm::IsaPred); if (!operand) return failure(); @@ -1156,8 +1155,8 @@ struct WarpOpCreateMask : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp, PatternRewriter &rewriter) const override { - OpOperand *yieldOperand = getWarpResult( - warpOp, [](Operation *op) { return isa(op); }); + OpOperand *yieldOperand = + getWarpResult(warpOp, llvm::IsaPred); if (!yieldOperand) return failure(); @@ -1222,8 +1221,8 @@ struct WarpOpExtract : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp, PatternRewriter &rewriter) const override { - OpOperand *operand = getWarpResult( - warpOp, [](Operation *op) { return isa(op); }); + OpOperand *operand = + getWarpResult(warpOp, llvm::IsaPred); if (!operand) return failure(); unsigned int operandNumber = operand->getOperandNumber(); @@ -1325,9 +1324,8 @@ struct WarpOpExtractElement : public OpRewritePattern { warpShuffleFromIdxFn(std::move(fn)) {} LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp, PatternRewriter &rewriter) const override { - OpOperand *operand = getWarpResult(warpOp, [](Operation *op) { - return isa(op); - }); + OpOperand *operand = + getWarpResult(warpOp, llvm::IsaPred); if (!operand) return failure(); unsigned int operandNumber = operand->getOperandNumber(); @@ -1422,8 +1420,8 @@ struct WarpOpInsertElement : public OpRewritePattern { LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp, PatternRewriter &rewriter) const override { - OpOperand *operand = getWarpResult( - warpOp, [](Operation *op) { return isa(op); }); + OpOperand *operand = + getWarpResult(warpOp, llvm::IsaPred); if (!operand) return failure(); unsigned int operandNumber = operand->getOperandNumber(); @@ -1503,8 +1501,7 @@ struct WarpOpInsert : public OpRewritePattern { LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp, PatternRewriter &rewriter) const override { - OpOperand *operand = getWarpResult( - warpOp, [](Operation *op) { return isa(op); }); + OpOperand *operand = getWarpResult(warpOp, llvm::IsaPred); if (!operand) return failure(); unsigned int operandNumber = operand->getOperandNumber(); @@ -1808,8 +1805,8 @@ struct WarpOpReduction : public OpRewritePattern { LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp, PatternRewriter &rewriter) const override { - OpOperand *yieldOperand = getWarpResult( - warpOp, [](Operation *op) { return isa(op); }); + OpOperand *yieldOperand = + getWarpResult(warpOp, llvm::IsaPred); if (!yieldOperand) return failure(); diff --git a/mlir/lib/IR/AffineMap.cpp b/mlir/lib/IR/AffineMap.cpp index 00a0f05b6333..6cdc2682753f 100644 --- a/mlir/lib/IR/AffineMap.cpp +++ b/mlir/lib/IR/AffineMap.cpp @@ -359,9 +359,7 @@ bool AffineMap::isSingleConstant() const { } bool AffineMap::isConstant() const { - return llvm::all_of(getResults(), [](AffineExpr expr) { - return isa(expr); - }); + return llvm::all_of(getResults(), llvm::IsaPred); } int64_t AffineMap::getSingleConstantResult() const { diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp index d6d59837d48a..ca5ff9f72e3e 100644 --- a/mlir/lib/IR/Operation.cpp +++ b/mlir/lib/IR/Operation.cpp @@ -1288,9 +1288,7 @@ LogicalResult OpTrait::impl::verifyNoRegionArguments(Operation *op) { } LogicalResult OpTrait::impl::verifyElementwise(Operation *op) { - auto isMappableType = [](Type type) { - return llvm::isa(type); - }; + auto isMappableType = llvm::IsaPred; auto resultMappableTypes = llvm::to_vector<1>( llvm::make_filter_range(op->getResultTypes(), isMappableType)); auto operandMappableTypes = llvm::to_vector<2>( diff --git a/mlir/lib/TableGen/Class.cpp b/mlir/lib/TableGen/Class.cpp index 9092adcc627c..fedf64fd96b0 100644 --- a/mlir/lib/TableGen/Class.cpp +++ b/mlir/lib/TableGen/Class.cpp @@ -369,9 +369,7 @@ void Class::finalize() { Visibility Class::getLastVisibilityDecl() const { auto reverseDecls = llvm::reverse(declarations); - auto it = llvm::find_if(reverseDecls, [](auto &decl) { - return isa(decl); - }); + auto it = llvm::find_if(reverseDecls, llvm::IsaPred); return it == reverseDecls.end() ? (isStruct ? Visibility::Public : Visibility::Private) : cast(**it).getVisibility(); diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp index 95c7af2f07be..0b07b4b06dfc 100644 --- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp +++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp @@ -1000,8 +1000,7 @@ static LogicalResult printOperation(CppEmitter &emitter, "with multiple blocks needs variables declared at top"); } - if (llvm::any_of(functionOp.getResultTypes(), - [](Type type) { return isa(type); })) { + if (llvm::any_of(functionOp.getResultTypes(), llvm::IsaPred)) { return functionOp.emitOpError() << "cannot emit array type as result type"; } @@ -1576,7 +1575,7 @@ LogicalResult CppEmitter::emitTypes(Location loc, ArrayRef types) { } LogicalResult CppEmitter::emitTupleType(Location loc, ArrayRef types) { - if (llvm::any_of(types, [](Type type) { return isa(type); })) { + if (llvm::any_of(types, llvm::IsaPred)) { return emitError(loc, "cannot emit tuple of array type"); } os << "std::tuple<"; diff --git a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp index 4a4e878d8af9..9a74ac115e95 100644 --- a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp +++ b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp @@ -1031,9 +1031,9 @@ Serializer::processBlock(Block *block, bool omitLabel, // into multiple basic blocks. If that's the case, we need to emit the merge // right now and then create new blocks for further serialization of the ops // in this block. - if (emitMerge && llvm::any_of(block->getOperations(), [](Operation &op) { - return isa(op); - })) { + if (emitMerge && + llvm::any_of(block->getOperations(), + llvm::IsaPred)) { if (failed(emitMerge())) return failure(); emitMerge = nullptr; @@ -1045,7 +1045,7 @@ Serializer::processBlock(Block *block, bool omitLabel, } // Process each op in this block except the terminator. - for (auto &op : llvm::make_range(block->begin(), std::prev(block->end()))) { + for (Operation &op : llvm::drop_end(*block)) { if (failed(processOperation(&op))) return failure(); } diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index 2ec0b964b304..8671c1008902 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -2825,9 +2825,9 @@ static void computeNecessaryMaterializations( } // Check to see if this is an argument materialization. - auto isBlockArg = [](Value v) { return isa(v); }; - if (llvm::any_of(op->getOperands(), isBlockArg) || - llvm::any_of(inverseMapping[op->getResult(0)], isBlockArg)) { + if (llvm::any_of(op->getOperands(), llvm::IsaPred) || + llvm::any_of(inverseMapping[op->getResult(0)], + llvm::IsaPred)) { mat->setMaterializationKind(MaterializationKind::Argument); } diff --git a/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp b/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp index 7cb957d5ec29..fef9d8eb0fef 100644 --- a/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp +++ b/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp @@ -392,8 +392,7 @@ applyPartialOneToNConversion(Operation *op, OneToNTypeConverter &typeConverter, // Argument materialization. assert(castKind == getCastKindName(CastKind::Argument) && "unexpected value of cast kind attribute"); - assert(llvm::all_of(operands, - [&](Value v) { return isa(v); })); + assert(llvm::all_of(operands, llvm::IsaPred)); maybeResult = typeConverter.materializeArgumentConversion( rewriter, castOp->getLoc(), resultTypes.front(), castOp.getOperands()); -- GitLab From 1351d17826e1efa3da3b29b6e345d44cb0ce3bc9 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 1 Apr 2024 08:52:35 -0700 Subject: [PATCH 0114/1690] [InstrFDO][TypeProf] Implement binary instrumentation and profile read/write (#66825) (The profile format change is split into a standalone change into https://github.com/llvm/llvm-project/pull/81691) * For InstrFDO value profiling, implement instrumentation and lowering for virtual table address. * This is controlled by `-enable-vtable-value-profiling` and off by default. * When the option is on, raw profiles will carry serialized `VTableProfData` structs and compressed vtables as payloads. * Implement profile reader and writer support * Raw profile reader is used by `llvm-profdata` but not compiler. Raw profile reader will construct InstrProfSymtab with symbol names, and map profiled runtime address to vtable symbols. * Indexed profile reader is used by `llvm-profdata` and compiler. When initialized, the reader stores a pointer to the beginning of in-memory compressed vtable names and the length of string. When used in `llvm-profdata`, reader decompress the string to show symbols of a profiled site. When used in compiler, string decompression doesn't happen since IR is used to construct InstrProfSymtab. * Indexed profile writer collects the list of vtable names, and stores that to index profiles. * Text profile reader and writer support are added but mostly follow the implementation for indirect-call value type. * `llvm-profdata show -show-vtables ` is implemented. rfc in https://discourse.llvm.org/t/rfc-dynamic-type-profiling-and-optimizations-in-llvm/74600#pick-instrumentation-points-and-instrument-runtime-types-7 --- .../Linux/instrprof-vtable-value-prof.cpp | 142 ++++ .../llvm/Analysis/IndirectCallVisitor.h | 62 +- llvm/include/llvm/ProfileData/InstrProf.h | 128 +++- .../llvm/ProfileData/InstrProfReader.h | 6 + .../llvm/ProfileData/InstrProfWriter.h | 4 + llvm/lib/ProfileData/InstrProf.cpp | 66 +- llvm/lib/ProfileData/InstrProfReader.cpp | 33 +- llvm/lib/ProfileData/InstrProfWriter.cpp | 29 +- .../Instrumentation/InstrProfiling.cpp | 157 ++++- .../Instrumentation/PGOInstrumentation.cpp | 14 + .../Instrumentation/ValueProfilePlugins.inc | 35 +- .../PGOProfile/vtable_prof_unsupported.ll | 34 + .../Transforms/PGOProfile/vtable_profile.ll | 98 +++ .../Inputs/vtable-value-prof.proftext | 74 +++ .../llvm-profdata/vtable-value-prof.test | 83 +++ llvm/tools/llvm-profdata/llvm-profdata.cpp | 26 + llvm/unittests/ProfileData/InstrProfTest.cpp | 620 ++++++++++++++---- 17 files changed, 1419 insertions(+), 192 deletions(-) create mode 100644 compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp create mode 100644 llvm/test/Transforms/PGOProfile/vtable_prof_unsupported.ll create mode 100644 llvm/test/Transforms/PGOProfile/vtable_profile.ll create mode 100644 llvm/test/tools/llvm-profdata/Inputs/vtable-value-prof.proftext create mode 100644 llvm/test/tools/llvm-profdata/vtable-value-prof.test diff --git a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp new file mode 100644 index 000000000000..5c8426b40892 --- /dev/null +++ b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp @@ -0,0 +1,142 @@ +// REQUIRES: lld-available + +// RUN: %clangxx_pgogen -fuse-ld=lld -O2 -g -fprofile-generate=. -mllvm -enable-vtable-value-profiling %s -o %t-test +// RUN: env LLVM_PROFILE_FILE=%t-test.profraw %t-test + +// Show vtable profiles from raw profile. +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-test.profraw | FileCheck %s --check-prefixes=COMMON,RAW + +// Generate indexed profile from raw profile and show the data. +// RUN: llvm-profdata merge %t-test.profraw -o %t-test.profdata +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-test.profdata | FileCheck %s --check-prefixes=COMMON,INDEXED + +// Generate text profile from raw and indexed profiles respectively and show the data. +// RUN: llvm-profdata merge --text %t-test.profraw -o %t-raw.proftext +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %t-raw.proftext | FileCheck %s --check-prefix=ICTEXT +// RUN: llvm-profdata merge --text %t-test.profdata -o %t-indexed.proftext +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %t-indexed.proftext | FileCheck %s --check-prefix=ICTEXT + +// Generate indexed profile from text profiles and show the data +// RUN: llvm-profdata merge --binary %t-raw.proftext -o %t-text.profraw +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-text.profraw | FileCheck %s --check-prefixes=COMMON,INDEXED +// RUN: llvm-profdata merge --binary %t-indexed.proftext -o %t-text.profdata +// RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t-text.profdata | FileCheck %s --check-prefixes=COMMON,INDEXED + +// COMMON: Counters: +// COMMON-NEXT: main: +// COMMON-NEXT: Hash: 0x0f9a16fe6d398548 +// COMMON-NEXT: Counters: 2 +// COMMON-NEXT: Indirect Call Site Count: 2 +// COMMON-NEXT: Number of instrumented vtables: 2 +// RAW: Indirect Target Results: +// RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%) +// RAW-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%) +// RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%) +// RAW-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%) +// RAW-NEXT: VTable Results: +// RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%) +// RAW-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +// RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%) +// RAW-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +// INDEXED: Indirect Target Results: +// INDEXED-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%) +// INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%) +// INDEXED-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%) +// INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%) +// INDEXED-NEXT: VTable Results: +// INDEXED-NEXT: [ 0, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +// INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%) +// INDEXED-NEXT: [ 1, {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +// INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%) +// COMMON: Instrumentation level: IR entry_first = 0 +// COMMON-NEXT: Functions shown: 1 +// COMMON-NEXT: Total functions: 6 +// COMMON-NEXT: Maximum function count: 1000 +// COMMON-NEXT: Maximum internal block count: 250 +// COMMON-NEXT: Statistics for indirect call sites profile: +// COMMON-NEXT: Total number of sites: 2 +// COMMON-NEXT: Total number of sites with values: 2 +// COMMON-NEXT: Total number of profiled values: 4 +// COMMON-NEXT: Value sites histogram: +// COMMON-NEXT: NumTargets, SiteCount +// COMMON-NEXT: 2, 2 +// COMMON-NEXT: Statistics for vtable profile: +// COMMON-NEXT: Total number of sites: 2 +// COMMON-NEXT: Total number of sites with values: 2 +// COMMON-NEXT: Total number of profiled values: 4 +// COMMON-NEXT: Value sites histogram: +// COMMON-NEXT: NumTargets, SiteCount +// COMMON-NEXT: 2, 2 + +// ICTEXT: :ir +// ICTEXT: main +// ICTEXT: # Func Hash: +// ICTEXT: 1124236338992350536 +// ICTEXT: # Num Counters: +// ICTEXT: 2 +// ICTEXT: # Counter Values: +// ICTEXT: 1000 +// ICTEXT: 1 +// ICTEXT: # Num Value Kinds: +// ICTEXT: 2 +// ICTEXT: # ValueKind = IPVK_IndirectCallTarget: +// ICTEXT: 0 +// ICTEXT: # NumValueSites: +// ICTEXT: 2 +// ICTEXT: 2 +// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func1Eii:750 +// ICTEXT: _ZN8Derived15func1Eii:250 +// ICTEXT: 2 +// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZN12_GLOBAL__N_18Derived25func2Eii:750 +// ICTEXT: _ZN8Derived15func2Eii:250 +// ICTEXT: # ValueKind = IPVK_VTableTarget: +// ICTEXT: 2 +// ICTEXT: # NumValueSites: +// ICTEXT: 2 +// ICTEXT: 2 +// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E:750 +// ICTEXT: _ZTV8Derived1:250 +// ICTEXT: 2 +// ICTEXT: {{.*}}instrprof-vtable-value-prof.cpp;_ZTVN12_GLOBAL__N_18Derived2E:750 +// ICTEXT: _ZTV8Derived1:250 + +#include +#include +class Base { +public: + virtual int func1(int a, int b) = 0; + virtual int func2(int a, int b) = 0; +}; +class Derived1 : public Base { +public: + int func1(int a, int b) override { return a + b; } + + int func2(int a, int b) override { return a * b; } +}; +namespace { +class Derived2 : public Base { +public: + int func1(int a, int b) override { return a - b; } + + int func2(int a, int b) override { return a * (a - b); } +}; +} // namespace +__attribute__((noinline)) Base *createType(int a) { + Base *base = nullptr; + if (a % 4 == 0) + base = new Derived1(); + else + base = new Derived2(); + return base; +} +int main(int argc, char **argv) { + int sum = 0; + for (int i = 0; i < 1000; i++) { + int a = rand(); + int b = rand(); + Base *ptr = createType(i); + sum += ptr->func1(a, b) + ptr->func2(b, a); + } + printf("sum is %d\n", sum); + return 0; +} diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h index 0825e19ecd2d..50815f4e3e83 100644 --- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h +++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h @@ -16,23 +16,75 @@ #include namespace llvm { -// Visitor class that finds all indirect call. +// Visitor class that finds indirect calls or instructions that gives vtable +// value, depending on Type. struct PGOIndirectCallVisitor : public InstVisitor { + enum class InstructionType { + kIndirectCall = 0, + kVTableVal = 1, + }; std::vector IndirectCalls; - PGOIndirectCallVisitor() = default; + std::vector ProfiledAddresses; + PGOIndirectCallVisitor(InstructionType Type) : Type(Type) {} void visitCallBase(CallBase &Call) { - if (Call.isIndirectCall()) + if (!Call.isIndirectCall()) + return; + + if (Type == InstructionType::kIndirectCall) { IndirectCalls.push_back(&Call); + return; + } + + assert(Type == InstructionType::kVTableVal && "Control flow guaranteed"); + + LoadInst *LI = dyn_cast(Call.getCalledOperand()); + // The code pattern to look for + // + // %vtable = load ptr, ptr %b + // %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1 + // %2 = load ptr, ptr %vfn + // %call = tail call i32 %2(ptr %b) + // + // %vtable is the vtable address value to profile, and + // %2 is the indirect call target address to profile. + if (LI != nullptr) { + Value *Ptr = LI->getPointerOperand(); + Value *VTablePtr = Ptr->stripInBoundsConstantOffsets(); + // This is a heuristic to find address feeding instructions. + // FIXME: Add support in the frontend so LLVM type intrinsics are + // emitted without LTO. This way, added intrinsics could filter + // non-vtable instructions and reduce instrumentation overhead. + // Since a non-vtable profiled address is not within the address + // range of vtable objects, it's stored as zero in indexed profiles. + // A pass that looks up symbol with an zero hash will (almost) always + // find nullptr and skip the actual transformation (e.g., comparison + // of symbols). So the performance overhead from non-vtable profiled + // address is negligible if exists at all. Comparing loaded address + // with symbol address guarantees correctness. + if (VTablePtr != nullptr && isa(VTablePtr)) + ProfiledAddresses.push_back(cast(VTablePtr)); + } } + +private: + InstructionType Type; }; -// Helper function that finds all indirect call sites. inline std::vector findIndirectCalls(Function &F) { - PGOIndirectCallVisitor ICV; + PGOIndirectCallVisitor ICV( + PGOIndirectCallVisitor::InstructionType::kIndirectCall); ICV.visit(F); return ICV.IndirectCalls; } + +inline std::vector findVTableAddrs(Function &F) { + PGOIndirectCallVisitor ICV( + PGOIndirectCallVisitor::InstructionType::kVTableVal); + ICV.visit(F); + return ICV.ProfiledAddresses; +} + } // namespace llvm #endif diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 3a71c02d6d5c..fd66c4ed948f 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -17,6 +17,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/IntervalMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" @@ -89,6 +90,9 @@ inline StringRef getInstrProfValueProfMemOpFuncName() { /// Return the name prefix of variables containing instrumented function names. inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; } +/// Return the name prefix of variables containing virtual table profile data. +inline StringRef getInstrProfVTableVarPrefix() { return "__profvt_"; } + /// Return the name prefix of variables containing per-function control data. inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; } @@ -106,9 +110,9 @@ inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; } /// Return the name of the variable holding the strings (possibly compressed) /// of all function's PGO names. -inline StringRef getInstrProfNamesVarName() { - return "__llvm_prf_nm"; -} +inline StringRef getInstrProfNamesVarName() { return "__llvm_prf_nm"; } + +inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; } /// Return the name of a covarage mapping variable (internal linkage) /// for each instrumented source module. Such variables are allocated @@ -140,7 +144,8 @@ inline StringRef getInstrProfRegFuncName() { return "__llvm_profile_register_function"; } -/// Return the name of the runtime interface that registers the PGO name strings. +/// Return the name of the runtime interface that registers the PGO name +/// strings. inline StringRef getInstrProfNamesRegFuncName() { return "__llvm_profile_register_names_function"; } @@ -246,6 +251,9 @@ Error collectGlobalObjectNameStrings(ArrayRef NameStrs, Error collectPGOFuncNameStrings(ArrayRef NameVars, std::string &Result, bool doCompression = true); +Error collectVTableStrings(ArrayRef VTables, + std::string &Result, bool doCompression); + /// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being /// set in IR PGO compilation. bool isIRPGOFlagSet(const Module *M); @@ -288,6 +296,8 @@ inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; } /// Return the PGOFuncName meta data associated with a function. MDNode *getPGOFuncNameMetadata(const Function &F); +std::string getPGOName(const GlobalVariable &V, bool InLTO = false); + /// Create the PGOFuncName meta data if PGOFuncName is different from /// function's raw name. This should only apply to internal linkage functions /// declared by users only. @@ -295,7 +305,7 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName); /// Check if we can use Comdat for profile variables. This will eliminate /// the duplicated profile variables for Comdat functions. -bool needsComdatForCounter(const Function &F, const Module &M); +bool needsComdatForCounter(const GlobalObject &GV, const Module &M); /// An enum describing the attributes of an instrumented profile. enum class InstrProfKind { @@ -431,23 +441,34 @@ public: using AddrHashMap = std::vector>; private: + using AddrIntervalMap = + IntervalMap>; StringRef Data; uint64_t Address = 0; - // Unique name strings. + // Unique name strings. Used to ensure entries in MD5NameMap (a vector that's + // going to be sorted) has unique MD5 keys in the first place. StringSet<> NameTab; + // Records the unique virtual table names. This is used by InstrProfWriter to + // write out an on-disk chained hash table of virtual table names. + // InstrProfWriter stores per function profile data (keyed by function names) + // so it doesn't use a StringSet for function names. + StringSet<> VTableNames; // A map from MD5 keys to function name strings. std::vector> MD5NameMap; + // A map from MD5 keys to function define. We only populate this map // when build the Symtab from a Module. std::vector> MD5FuncMap; // A map from function runtime address to function name MD5 hash. // This map is only populated and used by raw instr profile reader. AddrHashMap AddrToMD5Map; + + AddrIntervalMap::Allocator VTableAddrMapAllocator; + // This map is only populated and used by raw instr profile reader. + AddrIntervalMap VTableAddrMap; bool Sorted = false; - static StringRef getExternalSymbol() { - return "** External Symbol **"; - } + static StringRef getExternalSymbol() { return "** External Symbol **"; } // Returns the canonial name of the given PGOName. In a canonical name, all // suffixes that begins with "." except ".__uniq." are stripped. @@ -469,7 +490,7 @@ private: inline void finalizeSymtab(); public: - InstrProfSymtab() = default; + InstrProfSymtab() : VTableAddrMap(VTableAddrMapAllocator) {} // Not copyable or movable. // Consider std::unique_ptr for move. @@ -488,9 +509,19 @@ public: /// \c NameStrings is a string composed of one of more sub-strings /// encoded in the format described in \c collectPGOFuncNameStrings. - /// This method is a wrapper to \c readPGOFuncNameStrings method. + /// This method is a wrapper to \c readAndDecodeStrings method. Error create(StringRef NameStrings); + /// Initialize symtab states with function names and vtable names. \c + /// FuncNameStrings is a string composed of one or more encoded function name + /// strings, and \c VTableNameStrings composes of one or more encoded vtable + /// names. This interface is solely used by raw profile reader. + Error create(StringRef FuncNameStrings, StringRef VTableNameStrings); + + /// Initialize 'this' with the set of vtable names encoded in + /// \c CompressedVTableNames. + Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames); + /// This interface is used by reader of CoverageMapping test /// format. inline Error create(StringRef D, uint64_t BaseAddr); @@ -503,32 +534,69 @@ public: /// Create InstrProfSymtab from a set of names iteratable from /// \p IterRange. This interface is used by IndexedProfReader. - template Error create(const NameIterRange &IterRange); - - /// Update the symtab by adding \p FuncName to the table. This interface - /// is used by the raw and text profile readers. - Error addFuncName(StringRef FuncName) { - if (FuncName.empty()) + template + Error create(const NameIterRange &IterRange); + + /// Create InstrProfSymtab from a set of function names and vtable + /// names iteratable from \p IterRange. This interface is used by + /// IndexedProfReader. + template + Error create(const FuncNameIterRange &FuncIterRange, + const VTableNameIterRange &VTableIterRange); + + Error addSymbolName(StringRef SymbolName) { + if (SymbolName.empty()) return make_error(instrprof_error::malformed, - "function name is empty"); - auto Ins = NameTab.insert(FuncName); + "symbol name is empty"); + + // Insert into NameTab so that MD5NameMap (a vector that will be sorted) + // won't have duplicated entries in the first place. + auto Ins = NameTab.insert(SymbolName); if (Ins.second) { MD5NameMap.push_back(std::make_pair( - IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey())); + IndexedInstrProf::ComputeHash(SymbolName), Ins.first->getKey())); Sorted = false; } return Error::success(); } + /// The method name is kept since there are many callers. + /// It just forwards to 'addSymbolName'. + Error addFuncName(StringRef FuncName) { return addSymbolName(FuncName); } + + /// Adds VTableName as a known symbol, and inserts it to a map that + /// tracks all vtable names. + Error addVTableName(StringRef VTableName) { + if (Error E = addSymbolName(VTableName)) + return E; + + // Record VTableName. InstrProfWriter uses this set. The comment around + // class member explains why. + VTableNames.insert(VTableName); + return Error::success(); + } + + const StringSet<> &getVTableNames() const { return VTableNames; } + /// Map a function address to its name's MD5 hash. This interface /// is only used by the raw profiler reader. void mapAddress(uint64_t Addr, uint64_t MD5Val) { AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val)); } + /// Map the address range (i.e., [start_address, end_address)) of a variable + /// to its names' MD5 hash. This interface is only used by the raw profile + /// reader. + void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) { + VTableAddrMap.insert(StartAddr, EndAddr, MD5Val); + } + /// Return a function's hash, or 0, if the function isn't in this SymTab. uint64_t getFunctionHashFromAddress(uint64_t Address); + /// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab. + uint64_t getVTableHashFromAddress(uint64_t Address); + /// Return function's PGO name from the function name's symbol /// address in the object file. If an error occurs, return /// an empty string. @@ -574,6 +642,24 @@ Error InstrProfSymtab::create(const NameIterRange &IterRange) { return Error::success(); } +template +Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange, + const VTableNameIterRange &VTableIterRange) { + // Iterate elements by StringRef rather than by const reference. + // StringRef is small enough, so the loop is efficient whether + // element in the range is std::string or StringRef. + for (StringRef Name : FuncIterRange) + if (Error E = addFuncName(Name)) + return E; + + for (StringRef VTableName : VTableIterRange) + if (Error E = addVTableName(VTableName)) + return E; + + finalizeSymtab(); + return Error::success(); +} + void InstrProfSymtab::finalizeSymtab() { if (Sorted) return; @@ -877,6 +963,8 @@ private: return ValueData->IndirectCallSites; case IPVK_MemOPSize: return ValueData->MemOPSizes; + case IPVK_VTableTarget: + return ValueData->VTableTargets; default: llvm_unreachable("Unknown value kind!"); } diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index cfde5d3fc77d..e46570af3873 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -626,6 +626,12 @@ public: InstrProfKind getProfileKind() const override; Error populateSymtab(InstrProfSymtab &Symtab) override { + // FIXME: the create method calls 'finalizeSymtab' and sorts a bunch of + // arrays/maps. Since there are other data sources other than 'HashTable' to + // populate a symtab, it might make sense to have something like this + // 1. Let each data source populate Symtab and init the arrays/maps without + // calling 'finalizeSymtab' + // 2. Call 'finalizeSymtab' once to get all arrays/maps sorted if needed. return Symtab.create(HashTable->keys()); } }; diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h index d2156c867872..4b42392bc1e0 100644 --- a/llvm/include/llvm/ProfileData/InstrProfWriter.h +++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h @@ -63,6 +63,9 @@ private: // List of binary ids. std::vector BinaryIds; + // Read the vtable names from raw instr profile reader. + StringSet<> VTableNames; + // An enum describing the attributes of the profile. InstrProfKind ProfileKind = InstrProfKind::Unknown; // Use raw pointer here for the incomplete type object. @@ -95,6 +98,7 @@ public: void addRecord(NamedInstrProfRecord &&I, function_ref Warn) { addRecord(std::move(I), 1, Warn); } + void addVTableName(StringRef VTableName) { VTableNames.insert(VTableName); } /// Add \p SrcTraces using reservoir sampling where \p SrcStreamSize is the /// total number of temporal profiling traces the source has seen. diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 8cf97fcb1dab..90c3cfc45b98 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -34,6 +34,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Compression.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" @@ -57,6 +58,8 @@ using namespace llvm; +#define DEBUG_TYPE "instrprof" + static cl::opt StaticFuncFullModulePrefix( "static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " @@ -219,6 +222,12 @@ cl::opt DoInstrProfNameCompression( "enable-name-compression", cl::desc("Enable name/filename string compression"), cl::init(true)); +cl::opt EnableVTableValueProfiling( + "enable-vtable-value-profiling", cl::init(false), + cl::desc("If true, the virtual table address will be instrumented to know " + "the types of a C++ pointer. The information is used in indirect " + "call promotion to do selective vtable-based comparison.")); + std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo) { @@ -378,6 +387,13 @@ std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, ""); } +std::string getPGOName(const GlobalVariable &V, bool InLTO) { + // PGONameMetadata should be set by compiler at profile use time + // and read by symtab creation to look up symbols corresponding to + // a MD5 hash. + return getIRPGOObjectName(V, InLTO, /*PGONameMetadata=*/nullptr); +} + // See getIRPGOObjectName() for a discription of the format. std::pair getParsedIRPGOName(StringRef IRPGOName) { auto [FileName, MangledName] = IRPGOName.split(kGlobalIdentifierDelimiter); @@ -459,6 +475,7 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) { if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO))) return E; } + Sorted = false; finalizeSymtab(); return Error::success(); @@ -517,6 +534,25 @@ Error InstrProfSymtab::create(StringRef NameStrings) { std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1)); } +Error InstrProfSymtab::create(StringRef FuncNameStrings, + StringRef VTableNameStrings) { + if (Error E = readAndDecodeStrings(FuncNameStrings, + std::bind(&InstrProfSymtab::addFuncName, + this, std::placeholders::_1))) + return E; + + return readAndDecodeStrings( + VTableNameStrings, + std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1)); +} + +Error InstrProfSymtab::initVTableNamesFromCompressedStrings( + StringRef CompressedVTableStrings) { + return readAndDecodeStrings( + CompressedVTableStrings, + std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1)); +} + StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) { // In ThinLTO, local function may have been promoted to global and have // suffix ".llvm." added to the function name. We need to add the @@ -560,6 +596,12 @@ Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) { return Error::success(); } +uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) { + // Given a runtime address, look up the hash value in the interval map, and + // fallback to value 0 if a hash value is not found. + return VTableAddrMap.lookup(Address, 0); +} + uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) { finalizeSymtab(); auto It = partition_point(AddrToMD5Map, [=](std::pair A) { @@ -636,6 +678,16 @@ Error collectPGOFuncNameStrings(ArrayRef NameVars, NameStrs, compression::zlib::isAvailable() && doCompression, Result); } +Error collectVTableStrings(ArrayRef VTables, + std::string &Result, bool doCompression) { + std::vector VTableNameStrs; + for (auto *VTable : VTables) + VTableNameStrs.push_back(getPGOName(*VTable)); + return collectGlobalObjectNameStrings( + VTableNameStrs, compression::zlib::isAvailable() && doCompression, + Result); +} + void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const { uint64_t FuncSum = 0; Sum.NumEntries += Counts.size(); @@ -898,6 +950,9 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, if (ValueKind == IPVK_IndirectCallTarget) return SymTab->getFunctionHashFromAddress(Value); + if (ValueKind == IPVK_VTableTarget) + return SymTab->getVTableHashFromAddress(Value); + return Value; } @@ -1288,8 +1343,8 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { F.setMetadata(getPGOFuncNameMetadataName(), N); } -bool needsComdatForCounter(const Function &F, const Module &M) { - if (F.hasComdat()) +bool needsComdatForCounter(const GlobalObject &GO, const Module &M) { + if (GO.hasComdat()) return true; if (!Triple(M.getTargetTriple()).supportsCOMDAT()) @@ -1305,7 +1360,7 @@ bool needsComdatForCounter(const Function &F, const Module &M) { // available_externally functions will end up being duplicated in raw profile // data. This can result in distorted profile as the counts of those dups // will be accumulated by the profile merger. - GlobalValue::LinkageTypes Linkage = F.getLinkage(); + GlobalValue::LinkageTypes Linkage = GO.getLinkage(); if (Linkage != GlobalValue::ExternalWeakLinkage && Linkage != GlobalValue::AvailableExternallyLinkage) return false; @@ -1461,7 +1516,7 @@ void OverlapStats::dump(raw_fd_ostream &OS) const { for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f) continue; - char ProfileKindName[20]; + char ProfileKindName[20] = {0}; switch (I) { case IPVK_IndirectCallTarget: strncpy(ProfileKindName, "IndirectCall", 19); @@ -1469,6 +1524,9 @@ void OverlapStats::dump(raw_fd_ostream &OS) const { case IPVK_MemOPSize: strncpy(ProfileKindName, "MemOP", 19); break; + case IPVK_VTableTarget: + strncpy(ProfileKindName, "VTable", 19); + break; default: snprintf(ProfileKindName, 19, "VP[%d]", I); break; diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index a275d4852c15..7ac5c561dc08 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -370,8 +370,11 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { } else if (ValueKind == IPVK_VTableTarget) { if (InstrProfSymtab::isExternalSymbol(VD.first)) Value = 0; - else + else { + if (Error E = Symtab->addVTableName(VD.first)) + return E; Value = IndexedInstrProf::ComputeHash(VD.first); + } } else { READ_NUM(VD.first, Value); } @@ -539,7 +542,8 @@ Error RawInstrProfReader::readNextHeader(const char *CurrentPos) { template Error RawInstrProfReader::createSymtab(InstrProfSymtab &Symtab) { - if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart))) + if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart), + StringRef(VNamesStart, VNamesEnd - VNamesStart))) return error(std::move(E)); for (const RawInstrProf::ProfileData *I = Data; I != DataEnd; ++I) { const IntPtrT FPtr = swap(I->FunctionPointer); @@ -547,6 +551,21 @@ Error RawInstrProfReader::createSymtab(InstrProfSymtab &Symtab) { continue; Symtab.mapAddress(FPtr, swap(I->NameRef)); } + + if (VTableBegin != nullptr && VTableEnd != nullptr) { + for (const RawInstrProf::VTableProfileData *I = VTableBegin; + I != VTableEnd; ++I) { + const IntPtrT VPtr = swap(I->VTablePointer); + if (!VPtr) + continue; + // Map both begin and end address to the name hash, since the instrumented + // address could be somewhere in the middle. + // VPtr is of type uint32_t or uint64_t so 'VPtr + I->VTableSize' marks + // the end of vtable address. + Symtab.mapVTableAddress(VPtr, VPtr + swap(I->VTableSize), + swap(I->VTableNameHash)); + } + } return success(); } @@ -1397,7 +1416,15 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() { if (Symtab) return *Symtab; - std::unique_ptr NewSymtab = std::make_unique(); + auto NewSymtab = std::make_unique(); + + if (Error E = NewSymtab->initVTableNamesFromCompressedStrings( + StringRef(VTableNamePtr, CompressedVTableNamesLen))) { + auto [ErrCode, Msg] = InstrProfError::take(std::move(E)); + consumeError(error(ErrCode, Msg)); + } + + // finalizeSymtab is called inside populateSymtab. if (Error E = Index->populateSymtab(*NewSymtab)) { auto [ErrCode, Msg] = InstrProfError::take(std::move(E)); consumeError(error(ErrCode, Msg)); diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index 8f067f8d05e2..c2c94ba30c65 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -19,6 +19,7 @@ #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/MemProf.h" #include "llvm/ProfileData/ProfileCommon.h" +#include "llvm/Support/Compression.h" #include "llvm/Support/Endian.h" #include "llvm/Support/EndianStream.h" #include "llvm/Support/Error.h" @@ -636,13 +637,18 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { uint64_t VTableNamesSectionStart = OS.tell(); if (!WritePrevVersion) { - // Use a dummy (and uncompressed) string as compressed vtable names and get - // the necessary profile format change in place for version 12. - // TODO: Store the list of vtable names in InstrProfWriter and use the - // real compressed name. - std::string CompressedVTableNames = "VTableNames"; + std::vector VTableNameStrs; + for (StringRef VTableName : VTableNames.keys()) + VTableNameStrs.push_back(VTableName.str()); + + std::string CompressedVTableNames; + if (!VTableNameStrs.empty()) + if (Error E = collectGlobalObjectNameStrings( + VTableNameStrs, compression::zlib::isAvailable(), + CompressedVTableNames)) + return E; - uint64_t CompressedStringLen = CompressedVTableNames.length(); + const uint64_t CompressedStringLen = CompressedVTableNames.length(); // Record the length of compressed string. OS.write(CompressedStringLen); @@ -652,12 +658,11 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { OS.writeByte(static_cast(c)); // Pad up to a multiple of 8. - // InstrProfReader would read bytes according to 'CompressedStringLen'. - uint64_t PaddedLength = alignTo(CompressedStringLen, 8); + // InstrProfReader could read bytes according to 'CompressedStringLen'. + const uint64_t PaddedLength = alignTo(CompressedStringLen, 8); - for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) { + for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) OS.writeByte(0); - } } uint64_t TemporalProfTracesSectionStart = 0; @@ -866,6 +871,10 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) { } } + for (const auto &VTableName : VTableNames) + if (Error E = Symtab.addVTableName(VTableName.getKey())) + return E; + if (static_cast(ProfileKind & InstrProfKind::TemporalProfile)) writeTextTemporalProfTraceData(OS, Symtab); diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index c42c53edd511..f9b58d9f2782 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -64,6 +64,9 @@ using namespace llvm; #define DEBUG_TYPE "instrprof" namespace llvm { +// Command line option to enable vtable value profiling. Defined in +// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling= +extern cl::opt EnableVTableValueProfiling; // TODO: Remove -debug-info-correlate in next LLVM release, in favor of // -profile-correlate=debug-info. cl::opt DebugInfoCorrelate( @@ -219,12 +222,18 @@ private: PerFunctionProfileData() = default; }; DenseMap ProfileDataMap; + // Key is virtual table variable, value is 'VTableProfData' in the form of + // GlobalVariable. + DenseMap VTableDataMap; /// If runtime relocation is enabled, this maps functions to the load /// instruction that produces the profile relocation bias. DenseMap FunctionToProfileBiasMap; std::vector CompilerUsedVars; std::vector UsedVars; std::vector ReferencedNames; + // The list of virtual table variables of which the VTableProfData is + // collected. + std::vector ReferencedVTables; GlobalVariable *NamesVar = nullptr; size_t NamesSize = 0; @@ -308,7 +317,7 @@ private: GlobalValue::LinkageTypes Linkage); /// Set Comdat property of GV, if required. - void maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef VarName); + void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName); /// Setup the sections into which counters and bitmaps are allocated. GlobalVariable *setupProfileSection(InstrProfInstBase *Inc, @@ -317,9 +326,15 @@ private: /// Create INSTR_PROF_DATA variable for counters and bitmaps. void createDataVariable(InstrProfCntrInstBase *Inc); + /// Get the counters for virtual table values, creating them if necessary. + void getOrCreateVTableProfData(GlobalVariable *GV); + /// Emit the section with compressed function names. void emitNameData(); + /// Emit the section with compressed vtable names. + void emitVTableNames(); + /// Emit value nodes section for value profiling. void emitVNodes(); @@ -763,6 +778,12 @@ bool InstrLowerer::lower() { } } + if (EnableVTableValueProfiling) + for (GlobalVariable &GV : M.globals()) + // Global variables with type metadata are virtual table variables. + if (GV.hasMetadata(LLVMContext::MD_type)) + getOrCreateVTableProfData(&GV); + for (Function &F : M) MadeChange |= lowerIntrinsics(&F); @@ -776,6 +797,7 @@ bool InstrLowerer::lower() { emitVNodes(); emitNameData(); + emitVTableNames(); // Emit runtime hook for the cases where the target does not unconditionally // require pulling in profile runtime, and coverage is enabled on code that is @@ -1193,13 +1215,13 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { return true; } -void InstrLowerer::maybeSetComdat(GlobalVariable *GV, Function *Fn, +void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef CounterGroupName) { // Place lowered global variables in a comdat group if the associated function - // is a COMDAT. This will make sure that only one copy of global variable - // (e.g. function counters) of the COMDAT function will be emitted after - // linking. - bool NeedComdat = needsComdatForCounter(*Fn, M); + // or global variable is a COMDAT. This will make sure that only one copy of + // global variable (e.g. function counters) of the COMDAT function will be + // emitted after linking. + bool NeedComdat = needsComdatForCounter(*GO, M); bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); if (!UseComdat) @@ -1237,6 +1259,104 @@ void InstrLowerer::maybeSetComdat(GlobalVariable *GV, Function *Fn, GV->setLinkage(GlobalValue::InternalLinkage); } +static inline bool shouldRecordVTableAddr(GlobalVariable *GV) { + if (!profDataReferencedByCode(*GV->getParent())) + return false; + + if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() && + !GV->hasAvailableExternallyLinkage()) + return true; + + // This avoids the profile data from referencing internal symbols in + // COMDAT. + if (GV->hasLocalLinkage() && GV->hasComdat()) + return false; + + return true; +} + +// FIXME: Introduce an internal alias like what's done for functions to reduce +// the number of relocation entries. +static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) { + auto *Int8PtrTy = PointerType::getUnqual(GV->getContext()); + + // Store a nullptr in __profvt_ if a real address shouldn't be used. + if (!shouldRecordVTableAddr(GV)) + return ConstantPointerNull::get(Int8PtrTy); + + return ConstantExpr::getBitCast(GV, Int8PtrTy); +} + +void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) { + assert(!DebugInfoCorrelate && + "Value profiling is not supported with lightweight instrumentation"); + if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) + return; + + // Skip llvm internal global variable or __prof variables. + if (GV->getName().starts_with("llvm.") || + GV->getName().starts_with("__llvm") || + GV->getName().starts_with("__prof")) + return; + + // VTableProfData already created + auto It = VTableDataMap.find(GV); + if (It != VTableDataMap.end() && It->second) + return; + + GlobalValue::LinkageTypes Linkage = GV->getLinkage(); + GlobalValue::VisibilityTypes Visibility = GV->getVisibility(); + + // This is to keep consistent with per-function profile data + // for correctness. + if (TT.isOSBinFormatXCOFF()) { + Linkage = GlobalValue::InternalLinkage; + Visibility = GlobalValue::DefaultVisibility; + } + + LLVMContext &Ctx = M.getContext(); + Type *DataTypes[] = { +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType, +#include "llvm/ProfileData/InstrProfData.inc" +#undef INSTR_PROF_VTABLE_DATA + }; + + auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes)); + + // Used by INSTR_PROF_VTABLE_DATA MACRO + Constant *VTableAddr = getVTableAddrForProfData(GV); + const std::string PGOVTableName = getPGOName(*GV); + // Record the length of the vtable. This is needed since vtable pointers + // loaded from C++ objects might be from the middle of a vtable definition. + uint32_t VTableSizeVal = + M.getDataLayout().getTypeAllocSize(GV->getValueType()); + + Constant *DataVals[] = { +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init, +#include "llvm/ProfileData/InstrProfData.inc" +#undef INSTR_PROF_VTABLE_DATA + }; + + auto *Data = + new GlobalVariable(M, DataTy, /*constant=*/false, Linkage, + ConstantStruct::get(DataTy, DataVals), + getInstrProfVTableVarPrefix() + PGOVTableName); + + Data->setVisibility(Visibility); + Data->setSection(getInstrProfSectionName(IPSK_vtab, TT.getObjectFormat())); + Data->setAlignment(Align(8)); + + maybeSetComdat(Data, GV, Data->getName()); + + VTableDataMap[GV] = Data; + + ReferencedVTables.push_back(GV); + + // VTable is used by runtime but not referenced by other + // sections. Conservatively mark it linker retained. + UsedVars.push_back(Data); +} + GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc, InstrProfSectKind IPSK) { GlobalVariable *NamePtr = Inc->getName(); @@ -1633,6 +1753,31 @@ void InstrLowerer::emitNameData() { NamePtr->eraseFromParent(); } +void InstrLowerer::emitVTableNames() { + if (!EnableVTableValueProfiling || ReferencedVTables.empty()) + return; + + // Collect the PGO names of referenced vtables and compress them. + std::string CompressedVTableNames; + if (Error E = collectVTableStrings(ReferencedVTables, CompressedVTableNames, + DoInstrProfNameCompression)) { + report_fatal_error(Twine(toString(std::move(E))), false); + } + + auto &Ctx = M.getContext(); + auto *VTableNamesVal = ConstantDataArray::getString( + Ctx, StringRef(CompressedVTableNames), false /* AddNull */); + GlobalVariable *VTableNamesVar = + new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */, + GlobalValue::PrivateLinkage, VTableNamesVal, + getInstrProfVTableNamesVarName()); + VTableNamesVar->setSection( + getInstrProfSectionName(IPSK_vname, TT.getObjectFormat())); + VTableNamesVar->setAlignment(Align(1)); + // Make VTableNames linker retained. + UsedVars.push_back(VTableNamesVar); +} + void InstrLowerer::emitRegistration() { if (!needsRuntimeRegistrationOfSectionRange(TT)) return; diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index 50eccc69a38a..98c6f8cbf5af 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -327,6 +327,9 @@ extern cl::opt PGOViewCounts; // Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name= extern cl::opt ViewBlockFreqFuncName; +// Command line option to enable vtable value profiling. Defined in +// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling= +extern cl::opt EnableVTableValueProfiling; extern cl::opt ProfileCorrelate; } // namespace llvm @@ -581,6 +584,8 @@ public: NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size(); NumOfPGOBB += MST.bbInfoSize(); ValueSites[IPVK_IndirectCallTarget] = VPC.get(IPVK_IndirectCallTarget); + if (EnableVTableValueProfiling) + ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget); } else { NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size(); @@ -1775,6 +1780,15 @@ static bool InstrumentAllFunctions( // (before LTO/ThinLTO linking) to create these variables. if (!IsCS) createIRLevelProfileFlagVar(M, /*IsCS=*/false); + + Triple TT(M.getTargetTriple()); + LLVMContext &Ctx = M.getContext(); + if (!TT.isOSBinFormatELF() && EnableVTableValueProfiling) + Ctx.diagnose(DiagnosticInfoPGOProfile( + M.getName().data(), + Twine("VTable value profiling is presently not " + "supported for non-ELF object formats"), + DS_Warning)); std::unordered_multimap ComdatMembers; collectComdatMembers(M, ComdatMembers); diff --git a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc index 3a129de1acd0..b47ef8523ea1 100644 --- a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc +++ b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc @@ -90,9 +90,38 @@ public: } }; +///--------------------- VirtualTableValueProfilingPlugin -------------------- +class VTableProfilingPlugin { + Function &F; + +public: + static constexpr InstrProfValueKind Kind = IPVK_VTableTarget; + + VTableProfilingPlugin(Function &Fn, TargetLibraryInfo &TLI) : F(Fn) {} + + void run(std::vector &Candidates) { + std::vector Result = findVTableAddrs(F); + for (Instruction *I : Result) { + Instruction *InsertPt = I->getNextNonDebugInstruction(); + // When finding an insertion point, keep PHI and EH pad instructions + // before vp intrinsics. This is similar to + // `BasicBlock::getFirstInsertionPt`. + while (InsertPt && (dyn_cast(InsertPt) || InsertPt->isEHPad())) + InsertPt = InsertPt->getNextNonDebugInstruction(); + // Skip instrumentating the value if InsertPt is the last instruction. + // FIXME: Set InsertPt to the end of basic block to instrument the value + // if InsertPt is the last instruction. + if (InsertPt == nullptr) + continue; + + Instruction *AnnotatedInst = I; + Candidates.emplace_back(CandidateInfo{I, InsertPt, AnnotatedInst}); + } + } +}; + ///----------------------- Registration of the plugins ------------------------- /// For now, registering a plugin with the ValueProfileCollector is done by /// adding the plugin type to the VP_PLUGIN_LIST macro. -#define VP_PLUGIN_LIST \ - MemIntrinsicPlugin, \ - IndirectCallPromotionPlugin +#define VP_PLUGIN_LIST \ + MemIntrinsicPlugin, IndirectCallPromotionPlugin, VTableProfilingPlugin diff --git a/llvm/test/Transforms/PGOProfile/vtable_prof_unsupported.ll b/llvm/test/Transforms/PGOProfile/vtable_prof_unsupported.ll new file mode 100644 index 000000000000..f72a20fdc71a --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/vtable_prof_unsupported.ll @@ -0,0 +1,34 @@ +; RUN: opt < %s -passes=pgo-instr-gen -enable-vtable-value-profiling -S 2>&1 | FileCheck %s + +; Test that unsupported warning is emitted for non-ELF object files. +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-macosx14.0.0" + +; CHECK: warning: {{.*}} VTable value profiling is presently not supported for non-ELF object formats + +@_ZTV4Base = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN4Base4funcEi] }, !type !0, !type !1 +@_ZTV7Derived = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN7Derived4funcEi] }, !type !0, !type !1, !type !2, !type !3 + +@llvm.compiler.used = appending global [2 x ptr] [ptr @_ZTV4Base, ptr @_ZTV7Derived], section "llvm.metadata" + +define i32 @_Z4funci(i32 %a) { +entry: + %call = call ptr @_Z10createTypev() + %vtable = load ptr, ptr %call + %0 = call i1 @llvm.public.type.test(ptr %vtable, metadata !"_ZTS7Derived") + call void @llvm.assume(i1 %0) + %1 = load ptr, ptr %vtable + %call1 = call i32 %1(ptr %call, i32 %a) + ret i32 %call1 +} + +declare ptr @_Z10createTypev() +declare i1 @llvm.public.type.test(ptr, metadata) +declare void @llvm.assume(i1) +declare i32 @_ZN4Base4funcEi(ptr, i32) +declare i32 @_ZN7Derived4funcEi(ptr , i32) + +!0 = !{i64 16, !"_ZTS4Base"} +!1 = !{i64 16, !"_ZTSM4BaseFiiE.virtual"} +!2 = !{i64 16, !"_ZTS7Derived"} +!3 = !{i64 16, !"_ZTSM7DerivedFiiE.virtual"} diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll new file mode 100644 index 000000000000..a8440031e149 --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll @@ -0,0 +1,98 @@ +; RUN: opt < %s -passes=pgo-instr-gen -enable-vtable-value-profiling -S 2>&1 | FileCheck %s --check-prefix=GEN --implicit-check-not="VTable value profiling is presently not supported" +; RUN: opt < %s -passes=pgo-instr-gen,instrprof -enable-vtable-value-profiling -S 2>&1 | FileCheck %s --check-prefix=LOWER --implicit-check-not="VTable value profiling is presently not supported" + +; __llvm_prf_vnm stores zlib-compressed vtable names. +; REQUIRES: zlib + +source_filename = "vtable_local.ll" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; The test IR is generated based on the following C++ program. +; Base1 has external linkage and Base2 has local linkage. +; class Derived uses multiple inheritance so its virtual table +; global variable contains two vtables. func1 is loaded from +; the vtable compatible with class Base1, and func2 is loaded +; from the vtable compatible with class Base2. + +; class Base1 { +; public: +; virtual int func1(int a) ; +; }; +; +; namespace { +; class Base2 { +; public: +; __attribute__((noinline)) virtual int func2(int a) { +; return a; +; } +; }; +; } + +; class Derived : public Base1, public Base2 { +; public: +; Derived(int c) : v(c) {} +; private: +; int v; +; }; +; +; Derived* createType(); + +; int func(int a) { +; Derived* d = createType(); +; return d->func2(a) + d->func1(a); +; } + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@_ZTV7Derived = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base15func1Ei], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr @_ZN12_GLOBAL__N_15Base25func2Ei] }, !type !0, !type !3, !type !6, !type !8, !type !10 +@_ZTV5Base1 = available_externally constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base15func1Ei] }, !type !0 +@_ZTVN12_GLOBAL__N_15Base2E = internal constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN12_GLOBAL__N_15Base25func2Ei] }, !type !11, !type !8; !vcall_visibility !12 +@llvm.compiler.used = appending global [1 x ptr] [ptr @_ZTV5Base1], section "llvm.metadata" + +; GEN: __llvm_profile_raw_version = comdat any +; GEN: __llvm_profile_raw_version = hidden constant i64 72057594037927946, comdat +; GEN: __profn__Z4funci = private constant [8 x i8] c"_Z4funci" + +; LOWER: $__profvt__ZTV7Derived = comdat nodeduplicate +; LOWER: $"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E" = comdat nodeduplicate +; LOWER: @__profvt__ZTV7Derived = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8 +; LOWER: @"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E" = internal global { i64, ptr, i32 } { i64 1419990121885302679, ptr @_ZTVN12_GLOBAL__N_15Base2E, i32 24 }, section "__llvm_prf_vtab", comdat, align 8 +; LOWER: @__llvm_prf_vnm = private constant [64 x i8] c"7>x\DA\8B\8F\0A\093wI-\CA,KMa,+IL\CAI\8D\CF\C9ON\CC\D1\CB\C9\B1\8E\07J\FA\19\1A\C5\BB\FB\F8;9\FA\C4\C7\FB\C5\1B\9A:%\16\A7\1A\B9\02\00\19:\12o", section "__llvm_prf_vns", align 1 +; LOWER: @llvm.used = appending global [5 x ptr] [ptr @__profvt__ZTV7Derived, ptr @"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E", ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata" + +define i32 @_Z4funci(i32 %a) { +entry: + %call = call ptr @_Z10createTypev() + %add.ptr = getelementptr inbounds i8, ptr %call, i64 8 + %vtable = load ptr, ptr %add.ptr +; GEN: [[P1:%[0-9]+]] = ptrtoint ptr %vtable to i64 +; GEN: call void @llvm.instrprof.value.profile(ptr @__profn__Z4funci, i64 [[CFGHash:[0-9]+]], i64 [[P1]], i32 2, i32 0) +; LOWER: [[P1:%[0-9]+]] = ptrtoint ptr %vtable to i64 +; LOWER: call void @__llvm_profile_instrument_target(i64 [[P1]], ptr @__profd__Z4funci, i32 2) + %vfunc1 = load ptr, ptr %vtable + %call1 = call i32 %vfunc1(ptr %add.ptr, i32 %a) + %vtable2 = load ptr, ptr %call +; GEN: [[P2:%[0-9]+]] = ptrtoint ptr %vtable2 to i64 +; GEN: call void @llvm.instrprof.value.profile(ptr @__profn__Z4funci, i64 [[CFGHash]], i64 [[P2]], i32 2, i32 1) +; LOWER: [[P2:%[0-9]+]] = ptrtoint ptr %vtable2 to i64 +; LOWER: call void @__llvm_profile_instrument_target(i64 [[P2]], ptr @__profd__Z4funci, i32 3) + %vfunc2 = load ptr, ptr %vtable2 + %call4 = call i32 %vfunc2(ptr %call, i32 %a) + %add = add nsw i32 %call1, %call4 + ret i32 %add +} + +declare ptr @_Z10createTypev() +declare i32 @_ZN12_GLOBAL__N_15Base25func2Ei(ptr %this, i32 %a) +declare i32 @_ZN5Base15func1Ei(ptr, i32) + +!0 = !{i64 16, !"_ZTS5Base1"} +!3 = !{i64 16, !"_ZTS7Derived"} +!6 = !{i64 40, !7} +!7 = distinct !{} +!8 = !{i64 16, !9} +!9 = distinct !{} +!10 = !{i64 40, !9} +!11 = !{i64 16, !7} diff --git a/llvm/test/tools/llvm-profdata/Inputs/vtable-value-prof.proftext b/llvm/test/tools/llvm-profdata/Inputs/vtable-value-prof.proftext new file mode 100644 index 000000000000..372f9f97b164 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/Inputs/vtable-value-prof.proftext @@ -0,0 +1,74 @@ +# IR level Instrumentation Flag +:ir +_Z10createTypei +# Func Hash: +146835647075900052 +# Num Counters: +2 +# Counter Values: +750 +250 + +_ZN8Derived15func1Eii +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +250 + +_ZN8Derived15func2Eii +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +250 + +main +# Func Hash: +1124236338992350536 +# Num Counters: +2 +# Counter Values: +1000 +1 +# Num Value Kinds: +2 +# ValueKind = IPVK_IndirectCallTarget: +0 +# NumValueSites: +2 +2 +vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii:750 +_ZN8Derived15func1Eii:250 +2 +vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii:750 +_ZN8Derived15func2Eii:250 +# ValueKind = IPVK_VTableTarget: +2 +# NumValueSites: +2 +2 +vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750 +_ZTV8Derived1:250 +2 +vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750 +_ZTV8Derived1:250 + +vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +750 + +vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +750 + diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof.test b/llvm/test/tools/llvm-profdata/vtable-value-prof.test new file mode 100644 index 000000000000..378c2e11b236 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/vtable-value-prof.test @@ -0,0 +1,83 @@ +; RUN: rm -rf %t && mkdir %t && cd %t + +; Generate indexed profiles from text profiles +RUN: llvm-profdata merge %S/Inputs/vtable-value-prof.proftext -o indexed.profdata + +; Show indexed profiles +RUN: llvm-profdata show --function=main --ic-targets --show-vtables indexed.profdata | FileCheck %s --check-prefix=INDEXED + +; Show text profiles +RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %S/Inputs/vtable-value-prof.proftext | FileCheck %s --check-prefix=ICTEXT + +; Convert indexed profiles to its textual output and show it. +RUN: llvm-profdata merge --text -o text-from-indexed.proftext indexed.profdata +RUN: llvm-profdata show --function=main --ic-targets --show-vtables text-from-indexed.proftext | FileCheck %s --check-prefix=INDEXED +RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text text-from-indexed.proftext | FileCheck %s --check-prefix=ICTEXT + +INDEXED: Counters: +INDEXED-NEXT: main: +INDEXED-NEXT: Hash: 0x0f9a16fe6d398548 +INDEXED-NEXT: Counters: 2 +INDEXED-NEXT: Indirect Call Site Count: 2 +INDEXED-NEXT: Number of instrumented vtables: 2 +INDEXED-NEXT: Indirect Target Results: +INDEXED-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%) +INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%) +INDEXED-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%) +INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%) +INDEXED-NEXT: VTable Results: +INDEXED-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%) +INDEXED-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%) +INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%) +INDEXED-NEXT: Instrumentation level: IR entry_first = 0 +INDEXED-NEXT: Functions shown: 1 +INDEXED-NEXT: Total functions: 6 +INDEXED-NEXT: Maximum function count: 1000 +INDEXED-NEXT: Maximum internal block count: 250 +INDEXED-NEXT: Statistics for indirect call sites profile: +INDEXED-NEXT: Total number of sites: 2 +INDEXED-NEXT: Total number of sites with values: 2 +INDEXED-NEXT: Total number of profiled values: 4 +INDEXED-NEXT: Value sites histogram: +INDEXED-NEXT: NumTargets, SiteCount +INDEXED-NEXT: 2, 2 +INDEXED-NEXT: Statistics for vtable profile: +INDEXED-NEXT: Total number of sites: 2 +INDEXED-NEXT: Total number of sites with values: 2 +INDEXED-NEXT: Total number of profiled values: 4 +INDEXED-NEXT: Value sites histogram: +INDEXED-NEXT: NumTargets, SiteCount +INDEXED-NEXT: 2, 2 + +ICTEXT: :ir +ICTEXT: main +ICTEXT: # Func Hash: +ICTEXT: 1124236338992350536 +ICTEXT: # Num Counters: +ICTEXT: 2 +ICTEXT: # Counter Values: +ICTEXT: 1000 +ICTEXT: 1 +ICTEXT: # Num Value Kinds: +ICTEXT: 2 +ICTEXT: # ValueKind = IPVK_IndirectCallTarget: +ICTEXT: 0 +ICTEXT: # NumValueSites: +ICTEXT: 2 +ICTEXT: 2 +ICTEXT: {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii:750 +ICTEXT: _ZN8Derived15func1Eii:250 +ICTEXT: 2 +ICTEXT: {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii:750 +ICTEXT: _ZN8Derived15func2Eii:250 +ICTEXT: # ValueKind = IPVK_VTableTarget: +ICTEXT: 2 +ICTEXT: # NumValueSites: +ICTEXT: 2 +ICTEXT: 2 +ICTEXT: {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750 +ICTEXT: _ZTV8Derived1:250 +ICTEXT: 2 +ICTEXT: {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750 +ICTEXT: _ZTV8Derived1:250 diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index e8ee3c238194..0b78564ccea3 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -361,6 +361,9 @@ cl::opt ShowIndirectCallTargets( "ic-targets", cl::init(false), cl::desc("Show indirect call site target values for shown functions"), cl::sub(ShowSubcommand)); +cl::opt ShowVTables("show-vtables", cl::init(false), + cl::desc("Show vtable names for shown functions"), + cl::sub(ShowSubcommand)); cl::opt ShowMemOPSizes( "memop-sizes", cl::init(false), cl::desc("Show the profiled sizes of the memory intrinsic calls " @@ -739,6 +742,13 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, }); } + const InstrProfSymtab &symtab = Reader->getSymtab(); + const auto &VTableNames = symtab.getVTableNames(); + + for (const auto &kv : VTableNames) { + WC->Writer.addVTableName(kv.getKey()); + } + if (Reader->hasTemporalProfile()) { auto &Traces = Reader->getTemporalProfTraces(Input.Weight); if (!Traces.empty()) @@ -2834,6 +2844,10 @@ static int showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) { OS << " Indirect Call Site Count: " << Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n"; + if (ShowVTables) + OS << " Number of instrumented vtables: " + << Func.getNumValueSites(IPVK_VTableTarget) << "\n"; + uint32_t NumMemOPCalls = Func.getNumValueSites(IPVK_MemOPSize); if (ShowMemOPSizes && NumMemOPCalls > 0) OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls @@ -2855,6 +2869,13 @@ static int showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) { &(Reader->getSymtab())); } + if (ShowVTables) { + OS << " VTable Results:\n"; + traverseAllValueSites(Func, IPVK_VTableTarget, + VPStats[IPVK_VTableTarget], OS, + &(Reader->getSymtab())); + } + if (ShowMemOPSizes && NumMemOPCalls > 0) { OS << " Memory Intrinsic Size Results:\n"; traverseAllValueSites(Func, IPVK_MemOPSize, VPStats[IPVK_MemOPSize], OS, @@ -2903,6 +2924,11 @@ static int showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) { VPStats[IPVK_IndirectCallTarget]); } + if (ShownFunctions && ShowVTables) { + OS << "Statistics for vtable profile:\n"; + showValueSitesStats(OS, IPVK_VTableTarget, VPStats[IPVK_VTableTarget]); + } + if (ShownFunctions && ShowMemOPSizes) { OS << "Statistics for memory intrinsic calls sizes profile:\n"; showValueSitesStats(OS, IPVK_MemOPSize, VPStats[IPVK_MemOPSize]); diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index c9323420bda7..732f8fd792f8 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -638,32 +638,78 @@ TEST_F(InstrProfTest, test_irpgo_read_deprecated_names) { Succeeded()); } +// callee1 to callee6 are from vtable1 to vtable6 respectively. static const char callee1[] = "callee1"; static const char callee2[] = "callee2"; static const char callee3[] = "callee3"; static const char callee4[] = "callee4"; static const char callee5[] = "callee5"; static const char callee6[] = "callee6"; +// callee7 and callee8 are not from any vtables. +static const char callee7[] = "callee7"; +static const char callee8[] = "callee8"; +// 'callee' is primarily used to create multiple-element vtables. +static const char callee[] = "callee"; +static const uint64_t vtable1[] = {uint64_t(callee), uint64_t(callee1)}; +static const uint64_t vtable2[] = {uint64_t(callee2), uint64_t(callee)}; +static const uint64_t vtable3[] = { + uint64_t(callee), + uint64_t(callee3), +}; +static const uint64_t vtable4[] = {uint64_t(callee4), uint64_t(callee)}; +static const uint64_t vtable5[] = {uint64_t(callee5), uint64_t(callee)}; +static const uint64_t vtable6[] = {uint64_t(callee6), uint64_t(callee)}; + +// Returns the address of callee with a numbered suffix in vtable. +static uint64_t getCalleeAddress(const uint64_t *vtableAddr) { + uint64_t CalleeAddr; + // Callee with a numbered suffix is the 2nd element in vtable1 and vtable3, + // and the 1st element in the rest of vtables. + if (vtableAddr == vtable1 || vtableAddr == vtable3) + CalleeAddr = uint64_t(vtableAddr) + 8; + else + CalleeAddr = uint64_t(vtableAddr); + return CalleeAddr; +} -TEST_P(InstrProfReaderWriterTest, icall_data_read_write) { +TEST_P(InstrProfReaderWriterTest, icall_and_vtable_data_read_write) { NamedInstrProfRecord Record1("caller", 0x1234, {1, 2}); - // 4 value sites. - Record1.reserveSites(IPVK_IndirectCallTarget, 4); - InstrProfValueData VD0[] = { - {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}}; - Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr); - // No value profile data at the second site. - Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); - InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}}; - Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr); - InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}}; - Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); + // 4 indirect call value sites. + { + Record1.reserveSites(IPVK_IndirectCallTarget, 4); + InstrProfValueData VD0[] = { + {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}}; + Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr); + // No value profile data at the second site. + Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); + InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}}; + Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr); + InstrProfValueData VD3[] = {{(uint64_t)callee7, 1}, {(uint64_t)callee8, 2}}; + Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); + } + + // 2 vtable value sites. + { + InstrProfValueData VD0[] = { + {getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + {getCalleeAddress(vtable3), 3}, + }; + InstrProfValueData VD2[] = { + {getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + }; + Record1.addValueData(IPVK_VTableTarget, 0, VD0, 3, nullptr); + Record1.addValueData(IPVK_VTableTarget, 2, VD2, 2, nullptr); + } Writer.addRecord(std::move(Record1), getProfWeight(), Err); Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err); Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err); Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err); + Writer.addRecord({"callee7", 0x1235, {3, 4}}, Err); + Writer.addRecord({"callee8", 0x1235, {3, 4}}, Err); // Set writer value prof data endianness. Writer.setValueProfDataEndianness(getEndianness()); @@ -676,24 +722,63 @@ TEST_P(InstrProfReaderWriterTest, icall_data_read_write) { Expected R = Reader->getInstrProfRecord("caller", 0x1234); ASSERT_THAT_ERROR(R.takeError(), Succeeded()); + + // Test the number of instrumented indirect call sites and the number of + // profiled values at each site. ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget)); EXPECT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); EXPECT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); - EXPECT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + + // Test the number of instrumented vtable sites and the number of profiled + // values at each site. + ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 2U); + EXPECT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 0), 3U); + EXPECT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 1), 2U); + + // First indirect site. + { + uint64_t TotalC; + auto VD = R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC); + + EXPECT_EQ(VD[0].Count, 3U * getProfWeight()); + EXPECT_EQ(VD[1].Count, 2U * getProfWeight()); + EXPECT_EQ(VD[2].Count, 1U * getProfWeight()); + EXPECT_EQ(TotalC, 6U * getProfWeight()); + + EXPECT_STREQ((const char *)VD[0].Value, "callee3"); + EXPECT_STREQ((const char *)VD[1].Value, "callee2"); + EXPECT_STREQ((const char *)VD[2].Value, "callee1"); + } - uint64_t TotalC; - std::unique_ptr VD = - R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC); + // First vtable site. + { + uint64_t TotalC; + auto VD = R->getValueForSite(IPVK_VTableTarget, 0, &TotalC); + + EXPECT_EQ(VD[0].Count, 3U * getProfWeight()); + EXPECT_EQ(VD[1].Count, 2U * getProfWeight()); + EXPECT_EQ(VD[2].Count, 1U * getProfWeight()); + EXPECT_EQ(TotalC, 6U * getProfWeight()); + + EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD[2].Value, getCalleeAddress(vtable1)); + } + + // Second vtable site. + { + uint64_t TotalC; + auto VD = R->getValueForSite(IPVK_VTableTarget, 1, &TotalC); - EXPECT_EQ(3U * getProfWeight(), VD[0].Count); - EXPECT_EQ(2U * getProfWeight(), VD[1].Count); - EXPECT_EQ(1U * getProfWeight(), VD[2].Count); - EXPECT_EQ(6U * getProfWeight(), TotalC); + EXPECT_EQ(VD[0].Count, 2U * getProfWeight()); + EXPECT_EQ(VD[1].Count, 1U * getProfWeight()); + EXPECT_EQ(TotalC, 3U * getProfWeight()); - EXPECT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3")); - EXPECT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2")); - EXPECT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); + EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable1)); + } } INSTANTIATE_TEST_SUITE_P( @@ -801,33 +886,53 @@ TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) { ASSERT_EQ(1U, ValueData[3].Count); } -TEST_P(MaybeSparseInstrProfTest, icall_data_merge) { +TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) { static const char caller[] = "caller"; NamedInstrProfRecord Record11(caller, 0x1234, {1, 2}); NamedInstrProfRecord Record12(caller, 0x1234, {1, 2}); - // 5 value sites. - Record11.reserveSites(IPVK_IndirectCallTarget, 5); - InstrProfValueData VD0[] = {{uint64_t(callee1), 1}, - {uint64_t(callee2), 2}, - {uint64_t(callee3), 3}, - {uint64_t(callee4), 4}}; - Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr); + // 5 value sites for indirect calls. + { + Record11.reserveSites(IPVK_IndirectCallTarget, 5); + InstrProfValueData VD0[] = {{uint64_t(callee1), 1}, + {uint64_t(callee2), 2}, + {uint64_t(callee3), 3}, + {uint64_t(callee4), 4}}; + Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr); - // No value profile data at the second site. - Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); + // No value profile data at the second site. + Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr); - InstrProfValueData VD2[] = { - {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; - Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); + InstrProfValueData VD2[] = { + {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; + Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); - InstrProfValueData VD3[] = {{uint64_t(callee1), 1}}; - Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr); + InstrProfValueData VD3[] = {{uint64_t(callee7), 1}, {uint64_t(callee8), 2}}; + Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); - InstrProfValueData VD4[] = {{uint64_t(callee1), 1}, - {uint64_t(callee2), 2}, - {uint64_t(callee3), 3}}; - Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr); + InstrProfValueData VD4[] = { + {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; + Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr); + } + // 3 value sites for vtables. + { + Record11.reserveSites(IPVK_VTableTarget, 3); + InstrProfValueData VD0[] = {{getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + {getCalleeAddress(vtable3), 3}, + {getCalleeAddress(vtable4), 4}}; + Record11.addValueData(IPVK_VTableTarget, 0, VD0, 4, nullptr); + + InstrProfValueData VD2[] = {{getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + {getCalleeAddress(vtable3), 3}}; + Record11.addValueData(IPVK_VTableTarget, 1, VD2, 3, nullptr); + + InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + {getCalleeAddress(vtable3), 3}}; + Record11.addValueData(IPVK_VTableTarget, 3, VD4, 3, nullptr); + } // A different record for the same caller. Record12.reserveSites(IPVK_IndirectCallTarget, 5); @@ -843,11 +948,28 @@ TEST_P(MaybeSparseInstrProfTest, icall_data_merge) { Record12.addValueData(IPVK_IndirectCallTarget, 3, nullptr, 0, nullptr); - InstrProfValueData VD42[] = {{uint64_t(callee1), 1}, - {uint64_t(callee2), 2}, - {uint64_t(callee3), 3}}; + InstrProfValueData VD42[] = { + {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}}; Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, 3, nullptr); + // 3 value sites for vtables. + { + Record12.reserveSites(IPVK_VTableTarget, 3); + InstrProfValueData VD0[] = {{getCalleeAddress(vtable2), 5}, + {getCalleeAddress(vtable3), 3}}; + Record12.addValueData(IPVK_VTableTarget, 0, VD0, 2, nullptr); + + InstrProfValueData VD2[] = {{getCalleeAddress(vtable2), 1}, + {getCalleeAddress(vtable3), 3}, + {getCalleeAddress(vtable4), 4}}; + Record12.addValueData(IPVK_VTableTarget, 1, VD2, 3, nullptr); + + InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1}, + {getCalleeAddress(vtable2), 2}, + {getCalleeAddress(vtable3), 3}}; + Record12.addValueData(IPVK_VTableTarget, 3, VD4, 3, nullptr); + } + Writer.addRecord(std::move(Record11), Err); // Merge profile data. Writer.addRecord(std::move(Record12), Err); @@ -857,53 +979,95 @@ TEST_P(MaybeSparseInstrProfTest, icall_data_merge) { Writer.addRecord({callee3, 0x1235, {3, 4}}, Err); Writer.addRecord({callee3, 0x1235, {3, 4}}, Err); Writer.addRecord({callee4, 0x1235, {3, 5}}, Err); + Writer.addRecord({callee7, 0x1235, {3, 5}}, Err); + Writer.addRecord({callee8, 0x1235, {3, 5}}, Err); auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile)); + // Test the number of instrumented value sites and the number of profiled + // values for each site. Expected R = Reader->getInstrProfRecord("caller", 0x1234); EXPECT_THAT_ERROR(R.takeError(), Succeeded()); + // For indirect calls. ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget)); ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); - ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 4)); + // For vtables. + ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 3U); + ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 0), 4U); + ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); + ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); + + // Test the merged values for indirect calls. + { + auto VD = R->getValueForSite(IPVK_IndirectCallTarget, 0); + EXPECT_STREQ((const char *)VD[0].Value, "callee2"); + EXPECT_EQ(VD[0].Count, 7U); + EXPECT_STREQ((const char *)VD[1].Value, "callee3"); + EXPECT_EQ(VD[1].Count, 6U); + EXPECT_STREQ((const char *)VD[2].Value, "callee4"); + EXPECT_EQ(VD[2].Count, 4U); + EXPECT_STREQ((const char *)VD[3].Value, "callee1"); + EXPECT_EQ(VD[3].Count, 1U); + + auto VD_2(R->getValueForSite(IPVK_IndirectCallTarget, 2)); + EXPECT_STREQ((const char *)VD_2[0].Value, "callee3"); + EXPECT_EQ(VD_2[0].Count, 6U); + EXPECT_STREQ((const char *)VD_2[1].Value, "callee4"); + EXPECT_EQ(VD_2[1].Count, 4U); + EXPECT_STREQ((const char *)VD_2[2].Value, "callee2"); + EXPECT_EQ(VD_2[2].Count, 3U); + EXPECT_STREQ((const char *)VD_2[3].Value, "callee1"); + EXPECT_EQ(VD_2[3].Count, 1U); + + auto VD_3(R->getValueForSite(IPVK_IndirectCallTarget, 3)); + EXPECT_STREQ((const char *)VD_3[0].Value, "callee8"); + EXPECT_EQ(VD_3[0].Count, 2U); + EXPECT_STREQ((const char *)VD_3[1].Value, "callee7"); + EXPECT_EQ(VD_3[1].Count, 1U); + + auto VD_4(R->getValueForSite(IPVK_IndirectCallTarget, 4)); + EXPECT_STREQ((const char *)VD_4[0].Value, "callee3"); + EXPECT_EQ(VD_4[0].Count, 6U); + EXPECT_STREQ((const char *)VD_4[1].Value, "callee2"); + EXPECT_EQ(VD_4[1].Count, 4U); + EXPECT_STREQ((const char *)VD_4[2].Value, "callee1"); + EXPECT_EQ(VD_4[2].Count, 2U); + } - std::unique_ptr VD = - R->getValueForSite(IPVK_IndirectCallTarget, 0); - ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2")); - ASSERT_EQ(7U, VD[0].Count); - ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3")); - ASSERT_EQ(6U, VD[1].Count); - ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4")); - ASSERT_EQ(4U, VD[2].Count); - ASSERT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1")); - ASSERT_EQ(1U, VD[3].Count); - - std::unique_ptr VD_2( - R->getValueForSite(IPVK_IndirectCallTarget, 2)); - ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(6U, VD_2[0].Count); - ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4")); - ASSERT_EQ(4U, VD_2[1].Count); - ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2")); - ASSERT_EQ(3U, VD_2[2].Count); - ASSERT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1")); - ASSERT_EQ(1U, VD_2[3].Count); - - std::unique_ptr VD_3( - R->getValueForSite(IPVK_IndirectCallTarget, 3)); - ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee1")); - ASSERT_EQ(1U, VD_3[0].Count); - - std::unique_ptr VD_4( - R->getValueForSite(IPVK_IndirectCallTarget, 4)); - ASSERT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(6U, VD_4[0].Count); - ASSERT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2")); - ASSERT_EQ(4U, VD_4[1].Count); - ASSERT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1")); - ASSERT_EQ(2U, VD_4[2].Count); + // Test the merged values for vtables + { + auto VD0 = R->getValueForSite(IPVK_VTableTarget, 0); + EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD0[0].Count, 7U); + EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD0[1].Count, 6U); + EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable4)); + EXPECT_EQ(VD0[2].Count, 4U); + EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable1)); + EXPECT_EQ(VD0[3].Count, 1U); + + auto VD1 = R->getValueForSite(IPVK_VTableTarget, 1); + EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD1[0].Count, 6U); + EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable4)); + EXPECT_EQ(VD1[1].Count, 4U); + EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD1[2].Count, 3U); + EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable1)); + EXPECT_EQ(VD1[3].Count, 1U); + + auto VD2 = R->getValueForSite(IPVK_VTableTarget, 2); + EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD2[0].Count, 6U); + EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD2[1].Count, 4U); + EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable1)); + EXPECT_EQ(VD2[2].Count, 2U); + } } struct ValueProfileMergeEdgeCaseTest @@ -1027,30 +1191,62 @@ INSTANTIATE_TEST_SUITE_P( EdgeCaseTest, ValueProfileMergeEdgeCaseTest, ::testing::Combine(::testing::Bool(), /* Sparse */ ::testing::Values(IPVK_IndirectCallTarget, - IPVK_MemOPSize) /* ValueKind */ + IPVK_MemOPSize, + IPVK_VTableTarget) /* ValueKind */ )); static void addValueProfData(InstrProfRecord &Record) { - Record.reserveSites(IPVK_IndirectCallTarget, 5); - InstrProfValueData VD0[] = {{uint64_t(callee1), 400}, - {uint64_t(callee2), 1000}, - {uint64_t(callee3), 500}, - {uint64_t(callee4), 300}, - {uint64_t(callee5), 100}}; - Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr); - InstrProfValueData VD1[] = {{uint64_t(callee5), 800}, - {uint64_t(callee3), 1000}, - {uint64_t(callee2), 2500}, - {uint64_t(callee1), 1300}}; - Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr); - InstrProfValueData VD2[] = {{uint64_t(callee6), 800}, - {uint64_t(callee3), 1000}, - {uint64_t(callee4), 5500}}; - Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); - InstrProfValueData VD3[] = {{uint64_t(callee2), 1800}, - {uint64_t(callee3), 2000}}; - Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); - Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr); + // Add test data for indirect calls. + { + Record.reserveSites(IPVK_IndirectCallTarget, 6); + InstrProfValueData VD0[] = {{uint64_t(callee1), 400}, + {uint64_t(callee2), 1000}, + {uint64_t(callee3), 500}, + {uint64_t(callee4), 300}, + {uint64_t(callee5), 100}}; + Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr); + InstrProfValueData VD1[] = {{uint64_t(callee5), 800}, + {uint64_t(callee3), 1000}, + {uint64_t(callee2), 2500}, + {uint64_t(callee1), 1300}}; + Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr); + InstrProfValueData VD2[] = {{uint64_t(callee6), 800}, + {uint64_t(callee3), 1000}, + {uint64_t(callee4), 5500}}; + Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr); + InstrProfValueData VD3[] = {{uint64_t(callee2), 1800}, + {uint64_t(callee3), 2000}}; + Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr); + Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr); + InstrProfValueData VD5[] = {{uint64_t(callee7), 1234}, + {uint64_t(callee8), 5678}}; + Record.addValueData(IPVK_IndirectCallTarget, 5, VD5, 2, nullptr); + } + + // Add test data for vtables + { + Record.reserveSites(IPVK_VTableTarget, 4); + InstrProfValueData VD0[] = { + {getCalleeAddress(vtable1), 400}, {getCalleeAddress(vtable2), 1000}, + {getCalleeAddress(vtable3), 500}, {getCalleeAddress(vtable4), 300}, + {getCalleeAddress(vtable5), 100}, + }; + InstrProfValueData VD1[] = {{getCalleeAddress(vtable5), 800}, + {getCalleeAddress(vtable3), 1000}, + {getCalleeAddress(vtable2), 2500}, + {getCalleeAddress(vtable1), 1300}}; + InstrProfValueData VD2[] = { + {getCalleeAddress(vtable6), 800}, + {getCalleeAddress(vtable3), 1000}, + {getCalleeAddress(vtable4), 5500}, + }; + InstrProfValueData VD3[] = {{getCalleeAddress(vtable2), 1800}, + {getCalleeAddress(vtable3), 2000}}; + Record.addValueData(IPVK_VTableTarget, 0, VD0, 5, nullptr); + Record.addValueData(IPVK_VTableTarget, 1, VD1, 4, nullptr); + Record.addValueData(IPVK_VTableTarget, 2, VD2, 3, nullptr); + Record.addValueData(IPVK_VTableTarget, 3, VD3, 2, nullptr); + } } TEST(ValueProfileReadWriteTest, value_prof_data_read_write) { @@ -1063,59 +1259,107 @@ TEST(ValueProfileReadWriteTest, value_prof_data_read_write) { VPData->deserializeTo(Record, nullptr); // Now read data from Record and sanity check the data - ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget)); + ASSERT_EQ(6U, Record.getNumValueSites(IPVK_IndirectCallTarget)); ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4)); + ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 5)); auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { return VD1.Count > VD2.Count; }; + std::unique_ptr VD_0( Record.getValueForSite(IPVK_IndirectCallTarget, 0)); llvm::sort(&VD_0[0], &VD_0[5], Cmp); - ASSERT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2")); - ASSERT_EQ(1000U, VD_0[0].Count); - ASSERT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3")); - ASSERT_EQ(500U, VD_0[1].Count); - ASSERT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1")); - ASSERT_EQ(400U, VD_0[2].Count); - ASSERT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4")); - ASSERT_EQ(300U, VD_0[3].Count); - ASSERT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5")); - ASSERT_EQ(100U, VD_0[4].Count); + EXPECT_STREQ((const char *)VD_0[0].Value, "callee2"); + EXPECT_EQ(1000U, VD_0[0].Count); + EXPECT_STREQ((const char *)VD_0[1].Value, "callee3"); + EXPECT_EQ(500U, VD_0[1].Count); + EXPECT_STREQ((const char *)VD_0[2].Value, "callee1"); + EXPECT_EQ(400U, VD_0[2].Count); + EXPECT_STREQ((const char *)VD_0[3].Value, "callee4"); + EXPECT_EQ(300U, VD_0[3].Count); + EXPECT_STREQ((const char *)VD_0[4].Value, "callee5"); + EXPECT_EQ(100U, VD_0[4].Count); std::unique_ptr VD_1( Record.getValueForSite(IPVK_IndirectCallTarget, 1)); llvm::sort(&VD_1[0], &VD_1[4], Cmp); - ASSERT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2")); - ASSERT_EQ(2500U, VD_1[0].Count); - ASSERT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1")); - ASSERT_EQ(1300U, VD_1[1].Count); - ASSERT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3")); - ASSERT_EQ(1000U, VD_1[2].Count); - ASSERT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5")); - ASSERT_EQ(800U, VD_1[3].Count); + EXPECT_STREQ((const char *)VD_1[0].Value, "callee2"); + EXPECT_EQ(VD_1[0].Count, 2500U); + EXPECT_STREQ((const char *)VD_1[1].Value, "callee1"); + EXPECT_EQ(VD_1[1].Count, 1300U); + EXPECT_STREQ((const char *)VD_1[2].Value, "callee3"); + EXPECT_EQ(VD_1[2].Count, 1000U); + EXPECT_STREQ((const char *)VD_1[3].Value, "callee5"); + EXPECT_EQ(VD_1[3].Count, 800U); std::unique_ptr VD_2( Record.getValueForSite(IPVK_IndirectCallTarget, 2)); llvm::sort(&VD_2[0], &VD_2[3], Cmp); - ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4")); - ASSERT_EQ(5500U, VD_2[0].Count); - ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3")); - ASSERT_EQ(1000U, VD_2[1].Count); - ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6")); - ASSERT_EQ(800U, VD_2[2].Count); + EXPECT_STREQ((const char *)VD_2[0].Value, "callee4"); + EXPECT_EQ(VD_2[0].Count, 5500U); + EXPECT_STREQ((const char *)VD_2[1].Value, "callee3"); + EXPECT_EQ(VD_2[1].Count, 1000U); + EXPECT_STREQ((const char *)VD_2[2].Value, "callee6"); + EXPECT_EQ(VD_2[2].Count, 800U); std::unique_ptr VD_3( Record.getValueForSite(IPVK_IndirectCallTarget, 3)); llvm::sort(&VD_3[0], &VD_3[2], Cmp); - ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3")); - ASSERT_EQ(2000U, VD_3[0].Count); - ASSERT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2")); - ASSERT_EQ(1800U, VD_3[1].Count); + EXPECT_STREQ((const char *)VD_3[0].Value, "callee3"); + EXPECT_EQ(VD_3[0].Count, 2000U); + EXPECT_STREQ((const char *)VD_3[1].Value, "callee2"); + EXPECT_EQ(VD_3[1].Count, 1800U); + + ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 0), 5U); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 3), 2U); + + auto VD0(Record.getValueForSite(IPVK_VTableTarget, 0)); + llvm::sort(&VD0[0], &VD0[5], Cmp); + EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD0[0].Count, 1000U); + EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD0[1].Count, 500U); + EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable1)); + EXPECT_EQ(VD0[2].Count, 400U); + EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable4)); + EXPECT_EQ(VD0[3].Count, 300U); + EXPECT_EQ(VD0[4].Value, getCalleeAddress(vtable5)); + EXPECT_EQ(VD0[4].Count, 100U); + + auto VD1(Record.getValueForSite(IPVK_VTableTarget, 1)); + llvm::sort(&VD1[0], &VD1[4], Cmp); + EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD1[0].Count, 2500U); + EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable1)); + EXPECT_EQ(VD1[1].Count, 1300U); + EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD1[2].Count, 1000U); + EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable5)); + EXPECT_EQ(VD1[3].Count, 800U); + + auto VD2(Record.getValueForSite(IPVK_VTableTarget, 2)); + llvm::sort(&VD2[0], &VD2[3], Cmp); + EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable4)); + EXPECT_EQ(VD2[0].Count, 5500U); + EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD2[1].Count, 1000U); + EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable6)); + EXPECT_EQ(VD2[2].Count, 800U); + + auto VD3(Record.getValueForSite(IPVK_VTableTarget, 3)); + llvm::sort(&VD3[0], &VD3[2], Cmp); + EXPECT_EQ(VD3[0].Value, getCalleeAddress(vtable3)); + EXPECT_EQ(VD3[0].Count, 2000U); + EXPECT_EQ(VD3[1].Value, getCalleeAddress(vtable2)); + EXPECT_EQ(VD3[1].Count, 1800U); } TEST(ValueProfileReadWriteTest, symtab_mapping) { @@ -1132,27 +1376,121 @@ TEST(ValueProfileReadWriteTest, symtab_mapping) { Symtab.mapAddress(uint64_t(callee4), 0x4000ULL); // Missing mapping for callee5 + auto getVTableStartAddr = [](const uint64_t *vtable) -> uint64_t { + return uint64_t(vtable); + }; + auto getVTableEndAddr = [](const uint64_t *vtable) -> uint64_t { + return uint64_t(vtable) + 16; + }; + auto getVTableMidAddr = [](const uint64_t *vtable) -> uint64_t { + return uint64_t(vtable) + 8; + }; + // vtable1, vtable2, vtable3, vtable4 get mapped; vtable5, vtable6 are not + // mapped. + Symtab.mapVTableAddress(getVTableStartAddr(vtable1), + getVTableEndAddr(vtable1), MD5Hash("vtable1")); + Symtab.mapVTableAddress(getVTableStartAddr(vtable2), + getVTableEndAddr(vtable2), MD5Hash("vtable2")); + Symtab.mapVTableAddress(getVTableStartAddr(vtable3), + getVTableEndAddr(vtable3), MD5Hash("vtable3")); + Symtab.mapVTableAddress(getVTableStartAddr(vtable4), + getVTableEndAddr(vtable4), MD5Hash("vtable4")); + VPData->deserializeTo(Record, &Symtab); // Now read data from Record and sanity check the data - ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget)); - ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); + ASSERT_EQ(Record.getNumValueSites(IPVK_IndirectCallTarget), 6U); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0), 5U); + + // Look up the value correpsonding to the middle of a vtable in symtab and + // test that it's the hash of the name. + EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable1)), + MD5Hash("vtable1")); + EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable2)), + MD5Hash("vtable2")); + EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable3)), + MD5Hash("vtable3")); + EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable4)), + MD5Hash("vtable4")); auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { return VD1.Count > VD2.Count; }; - std::unique_ptr VD_0( - Record.getValueForSite(IPVK_IndirectCallTarget, 0)); + auto VD_0(Record.getValueForSite(IPVK_IndirectCallTarget, 0)); llvm::sort(&VD_0[0], &VD_0[5], Cmp); ASSERT_EQ(VD_0[0].Value, 0x2000ULL); - ASSERT_EQ(1000U, VD_0[0].Count); + ASSERT_EQ(VD_0[0].Count, 1000U); ASSERT_EQ(VD_0[1].Value, 0x3000ULL); - ASSERT_EQ(500U, VD_0[1].Count); + ASSERT_EQ(VD_0[1].Count, 500U); ASSERT_EQ(VD_0[2].Value, 0x1000ULL); - ASSERT_EQ(400U, VD_0[2].Count); + ASSERT_EQ(VD_0[2].Count, 400U); // callee5 does not have a mapped value -- default to 0. ASSERT_EQ(VD_0[4].Value, 0ULL); + + // Sanity check the vtable value data + ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U); + + { + // The first vtable site. + auto VD(Record.getValueForSite(IPVK_VTableTarget, 0)); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 0), 5U); + llvm::sort(&VD[0], &VD[5], Cmp); + EXPECT_EQ(VD[0].Count, 1000U); + EXPECT_EQ(VD[0].Value, MD5Hash("vtable2")); + EXPECT_EQ(VD[1].Count, 500U); + EXPECT_EQ(VD[1].Value, MD5Hash("vtable3")); + EXPECT_EQ(VD[2].Value, MD5Hash("vtable1")); + EXPECT_EQ(VD[2].Count, 400U); + EXPECT_EQ(VD[3].Value, MD5Hash("vtable4")); + EXPECT_EQ(VD[3].Count, 300U); + + // vtable5 isn't mapped -- default to 0. + EXPECT_EQ(VD[4].Value, 0U); + EXPECT_EQ(VD[4].Count, 100U); + } + + { + // The second vtable site. + auto VD(Record.getValueForSite(IPVK_VTableTarget, 1)); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); + llvm::sort(&VD[0], &VD[4], Cmp); + EXPECT_EQ(VD[0].Value, MD5Hash("vtable2")); + EXPECT_EQ(VD[0].Count, 2500U); + EXPECT_EQ(VD[1].Value, MD5Hash("vtable1")); + EXPECT_EQ(VD[1].Count, 1300U); + + EXPECT_EQ(VD[2].Value, MD5Hash("vtable3")); + EXPECT_EQ(VD[2].Count, 1000U); + // vtable5 isn't mapped -- default to 0. + EXPECT_EQ(VD[3].Value, 0U); + EXPECT_EQ(VD[3].Count, 800U); + } + + { + // The third vtable site. + auto VD(Record.getValueForSite(IPVK_VTableTarget, 2)); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); + llvm::sort(&VD[0], &VD[3], Cmp); + EXPECT_EQ(VD[0].Count, 5500U); + EXPECT_EQ(VD[0].Value, MD5Hash("vtable4")); + EXPECT_EQ(VD[1].Count, 1000U); + EXPECT_EQ(VD[1].Value, MD5Hash("vtable3")); + // vtable6 isn't mapped -- default to 0. + EXPECT_EQ(VD[2].Value, 0U); + EXPECT_EQ(VD[2].Count, 800U); + } + + { + // The fourth vtable site. + auto VD(Record.getValueForSite(IPVK_VTableTarget, 3)); + ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 3), 2U); + llvm::sort(&VD[0], &VD[2], Cmp); + EXPECT_EQ(VD[0].Count, 2000U); + EXPECT_EQ(VD[0].Value, MD5Hash("vtable3")); + EXPECT_EQ(VD[1].Count, 1800U); + EXPECT_EQ(VD[1].Value, MD5Hash("vtable2")); + } } TEST_P(MaybeSparseInstrProfTest, get_max_function_count) { @@ -1278,13 +1616,13 @@ TEST(SymtabTest, instr_prof_symtab_module_test) { std::string IRPGOName = getIRPGOFuncName(*F); auto IRPGOFuncName = ProfSymtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(IRPGOName)); - EXPECT_EQ(StringRef(IRPGOName), IRPGOFuncName); - EXPECT_EQ(StringRef(Funcs[I]), getParsedIRPGOName(IRPGOFuncName).second); + EXPECT_EQ(IRPGOName, IRPGOFuncName); + EXPECT_EQ(Funcs[I], getParsedIRPGOName(IRPGOFuncName).second); // Ensure we can still read this old record name. std::string PGOName = getPGOFuncName(*F); auto PGOFuncName = ProfSymtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(PGOName)); - EXPECT_EQ(StringRef(PGOName), PGOFuncName); + EXPECT_EQ(PGOName, PGOFuncName); EXPECT_THAT(PGOFuncName.str(), EndsWith(Funcs[I].str())); } } -- GitLab From 985c1a44f8d49e0afeba907fe29d881c19b319fc Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Mon, 1 Apr 2024 18:21:51 +0200 Subject: [PATCH 0115/1690] [libc++] Optimize the two range overload of mismatch (#86853) ``` ----------------------------------------------------------------------------- Benchmark old new ----------------------------------------------------------------------------- bm_mismatch_two_range_overload/1 0.941 ns 1.88 ns bm_mismatch_two_range_overload/2 1.43 ns 2.15 ns bm_mismatch_two_range_overload/3 1.95 ns 2.55 ns bm_mismatch_two_range_overload/4 2.58 ns 2.90 ns bm_mismatch_two_range_overload/5 3.75 ns 3.31 ns bm_mismatch_two_range_overload/6 5.00 ns 3.83 ns bm_mismatch_two_range_overload/7 5.59 ns 4.35 ns bm_mismatch_two_range_overload/8 6.37 ns 4.84 ns bm_mismatch_two_range_overload/16 11.8 ns 6.72 ns bm_mismatch_two_range_overload/64 45.5 ns 2.59 ns bm_mismatch_two_range_overload/512 366 ns 12.6 ns bm_mismatch_two_range_overload/4096 2890 ns 91.6 ns bm_mismatch_two_range_overload/32768 23038 ns 758 ns bm_mismatch_two_range_overload/262144 142813 ns 6573 ns bm_mismatch_two_range_overload/1048576 366679 ns 26710 ns bm_mismatch_two_range_overload/1 0.934 ns 1.88 ns bm_mismatch_two_range_overload/2 1.30 ns 2.58 ns bm_mismatch_two_range_overload/3 1.76 ns 3.28 ns bm_mismatch_two_range_overload/4 2.24 ns 3.98 ns bm_mismatch_two_range_overload/5 2.80 ns 4.92 ns bm_mismatch_two_range_overload/6 3.58 ns 6.01 ns bm_mismatch_two_range_overload/7 4.29 ns 7.03 ns bm_mismatch_two_range_overload/8 4.67 ns 7.39 ns bm_mismatch_two_range_overload/16 9.86 ns 13.1 ns bm_mismatch_two_range_overload/64 38.9 ns 4.55 ns bm_mismatch_two_range_overload/512 348 ns 27.7 ns bm_mismatch_two_range_overload/4096 2881 ns 225 ns bm_mismatch_two_range_overload/32768 23111 ns 1715 ns bm_mismatch_two_range_overload/262144 184846 ns 14416 ns bm_mismatch_two_range_overload/1048576 742885 ns 57264 ns bm_mismatch_two_range_overload/1 0.838 ns 1.19 ns bm_mismatch_two_range_overload/2 1.19 ns 1.65 ns bm_mismatch_two_range_overload/3 1.83 ns 2.06 ns bm_mismatch_two_range_overload/4 2.38 ns 2.42 ns bm_mismatch_two_range_overload/5 3.60 ns 2.47 ns bm_mismatch_two_range_overload/6 3.68 ns 3.05 ns bm_mismatch_two_range_overload/7 4.32 ns 3.36 ns bm_mismatch_two_range_overload/8 5.18 ns 3.58 ns bm_mismatch_two_range_overload/16 10.6 ns 2.84 ns bm_mismatch_two_range_overload/64 39.0 ns 7.78 ns bm_mismatch_two_range_overload/512 247 ns 53.9 ns bm_mismatch_two_range_overload/4096 1927 ns 429 ns bm_mismatch_two_range_overload/32768 15569 ns 3393 ns bm_mismatch_two_range_overload/262144 125413 ns 28504 ns bm_mismatch_two_range_overload/1048576 504549 ns 112729 ns ``` --- .../benchmarks/algorithms/mismatch.bench.cpp | 16 +++++++++ libcxx/include/__algorithm/mismatch.h | 34 ++++++++++++++++--- libcxx/include/__algorithm/ranges_mismatch.h | 22 +++++++----- libcxx/include/__algorithm/simd_utils.h | 9 ++++- .../test/libcxx/transitive_includes/cxx23.csv | 1 + .../test/libcxx/transitive_includes/cxx26.csv | 1 + .../mismatch/mismatch.pass.cpp | 8 ++--- 7 files changed, 74 insertions(+), 17 deletions(-) diff --git a/libcxx/benchmarks/algorithms/mismatch.bench.cpp b/libcxx/benchmarks/algorithms/mismatch.bench.cpp index 06289068bb04..791782879011 100644 --- a/libcxx/benchmarks/algorithms/mismatch.bench.cpp +++ b/libcxx/benchmarks/algorithms/mismatch.bench.cpp @@ -37,4 +37,20 @@ BENCHMARK(bm_mismatch)->Apply(BenchmarkSizes); BENCHMARK(bm_mismatch)->Apply(BenchmarkSizes); BENCHMARK(bm_mismatch)->Apply(BenchmarkSizes); +template +static void bm_mismatch_two_range_overload(benchmark::State& state) { + std::vector vec1(state.range(), '1'); + std::vector vec2(state.range(), '1'); + std::mt19937_64 rng(std::random_device{}()); + + vec1.back() = '2'; + for (auto _ : state) { + benchmark::DoNotOptimize(vec1); + benchmark::DoNotOptimize(std::mismatch(vec1.begin(), vec1.end(), vec2.begin(), vec2.end())); + } +} +BENCHMARK(bm_mismatch_two_range_overload)->DenseRange(1, 8)->Range(16, 1 << 20); +BENCHMARK(bm_mismatch_two_range_overload)->DenseRange(1, 8)->Range(16, 1 << 20); +BENCHMARK(bm_mismatch_two_range_overload)->DenseRange(1, 8)->Range(16, 1 << 20); + BENCHMARK_MAIN(); diff --git a/libcxx/include/__algorithm/mismatch.h b/libcxx/include/__algorithm/mismatch.h index 1cb83b01b2eb..8abb273ac178 100644 --- a/libcxx/include/__algorithm/mismatch.h +++ b/libcxx/include/__algorithm/mismatch.h @@ -11,6 +11,7 @@ #define _LIBCPP___ALGORITHM_MISMATCH_H #include <__algorithm/comp.h> +#include <__algorithm/min.h> #include <__algorithm/simd_utils.h> #include <__algorithm/unwrap_iter.h> #include <__config> @@ -136,6 +137,25 @@ mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __fi } #if _LIBCPP_STD_VER >= 14 +template +[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2> __mismatch( + _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { + while (__first1 != __last1 && __first2 != __last2) { + if (!std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2))) + break; + ++__first1; + ++__first2; + } + return {std::move(__first1), std::move(__first2)}; +} + +template +[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*> +__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Tp* __last2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { + auto __len = std::min(__last1 - __first1, __last2 - __first2); + return std::__mismatch(__first1, __first1 + __len, __first2, __pred, __proj1, __proj2); +} + template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> mismatch(_InputIterator1 __first1, @@ -143,10 +163,16 @@ mismatch(_InputIterator1 __first1, _InputIterator2 __first2, _InputIterator2 __last2, _BinaryPredicate __pred) { - for (; __first1 != __last1 && __first2 != __last2; ++__first1, (void)++__first2) - if (!__pred(*__first1, *__first2)) - break; - return pair<_InputIterator1, _InputIterator2>(__first1, __first2); + __identity __proj; + auto __res = std::__mismatch( + std::__unwrap_iter(__first1), + std::__unwrap_iter(__last1), + std::__unwrap_iter(__first2), + std::__unwrap_iter(__last2), + __pred, + __proj, + __proj); + return {std::__rewrap_iter(__first1, __res.first), std::__rewrap_iter(__first2, __res.second)}; } template diff --git a/libcxx/include/__algorithm/ranges_mismatch.h b/libcxx/include/__algorithm/ranges_mismatch.h index 037af3912623..d8a7dd43af09 100644 --- a/libcxx/include/__algorithm/ranges_mismatch.h +++ b/libcxx/include/__algorithm/ranges_mismatch.h @@ -10,6 +10,8 @@ #define _LIBCPP___ALGORITHM_RANGES_MISMATCH_H #include <__algorithm/in_in_result.h> +#include <__algorithm/mismatch.h> +#include <__algorithm/unwrap_range.h> #include <__config> #include <__functional/identity.h> #include <__functional/invoke.h> @@ -42,13 +44,17 @@ struct __fn { template static _LIBCPP_HIDE_FROM_ABI constexpr mismatch_result<_I1, _I2> __go(_I1 __first1, _S1 __last1, _I2 __first2, _S2 __last2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { - while (__first1 != __last1 && __first2 != __last2) { - if (!std::invoke(__pred, std::invoke(__proj1, *__first1), std::invoke(__proj2, *__first2))) - break; - ++__first1; - ++__first2; + if constexpr (forward_iterator<_I1> && forward_iterator<_I2>) { + auto __range1 = std::__unwrap_range(__first1, __last1); + auto __range2 = std::__unwrap_range(__first2, __last2); + auto __res = + std::__mismatch(__range1.first, __range1.second, __range2.first, __range2.second, __pred, __proj1, __proj2); + return {std::__rewrap_range<_S1>(__first1, __res.first), std::__rewrap_range<_S2>(__first2, __res.second)}; + } else { + auto __res = std::__mismatch( + std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2); + return {std::move(__res.first), std::move(__res.second)}; } - return {std::move(__first1), std::move(__first2)}; } template requires indirectly_comparable, iterator_t<_R2>, _Pred, _Proj1, _Proj2> - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr mismatch_result, - borrowed_iterator_t<_R2>> + _LIBCPP_NODISCARD_EXT + _LIBCPP_HIDE_FROM_ABI constexpr mismatch_result, borrowed_iterator_t<_R2>> operator()(_R1&& __r1, _R2&& __r2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const { return __go( ranges::begin(__r1), ranges::end(__r1), ranges::begin(__r2), ranges::end(__r2), __pred, __proj1, __proj2); diff --git a/libcxx/include/__algorithm/simd_utils.h b/libcxx/include/__algorithm/simd_utils.h index 1aedb3db010f..989a1957987e 100644 --- a/libcxx/include/__algorithm/simd_utils.h +++ b/libcxx/include/__algorithm/simd_utils.h @@ -9,6 +9,7 @@ #ifndef _LIBCPP___ALGORITHM_SIMD_UTILS_H #define _LIBCPP___ALGORITHM_SIMD_UTILS_H +#include <__algorithm/min.h> #include <__bit/bit_cast.h> #include <__bit/countr.h> #include <__config> @@ -22,6 +23,9 @@ # pragma GCC system_header #endif +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + // TODO: Find out how altivec changes things and allow vectorizations there too. #if _LIBCPP_STD_VER >= 14 && defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1700 && !defined(__ALTIVEC__) # define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 1 @@ -94,7 +98,8 @@ _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_T // This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876 auto __impl = [&](_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept { - return std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec))); + return std::min( + _Np, std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec)))); }; if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) { @@ -120,4 +125,6 @@ _LIBCPP_END_NAMESPACE_STD #endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS +_LIBCPP_POP_MACROS + #endif // _LIBCPP___ALGORITHM_SIMD_UTILS_H diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv index 79c67dc00cfb..69429b5bce82 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx23.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv @@ -452,6 +452,7 @@ random vector random version ranges compare ranges cstddef +ranges cstdint ranges cwchar ranges initializer_list ranges iterator diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv index 79c67dc00cfb..69429b5bce82 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx26.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv @@ -452,6 +452,7 @@ random vector random version ranges compare ranges cstddef +ranges cstdint ranges cwchar ranges initializer_list ranges iterator diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp index 55c9eea863c3..eb5f7cacdde3 100644 --- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp +++ b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp @@ -187,8 +187,8 @@ int main(int, char**) { { // check the tail of the vectorized loop for (size_t vec_size = 1; vec_size != 256; ++vec_size) { { - std::vector lhs(256); - std::vector rhs(256); + std::vector lhs(vec_size); + std::vector rhs(vec_size); check(lhs, rhs, lhs.size()); lhs.back() = 1; @@ -199,8 +199,8 @@ int main(int, char**) { rhs.back() = 0; } { - std::vector lhs(256); - std::vector rhs(256); + std::vector lhs(vec_size); + std::vector rhs(vec_size); check(lhs, rhs, lhs.size()); lhs.back() = 1; -- GitLab From a8cfa7cbdf6cc1a94ed25c90897d2e031f77a5a9 Mon Sep 17 00:00:00 2001 From: Jakub Kuderski Date: Mon, 1 Apr 2024 12:32:23 -0400 Subject: [PATCH 0116/1690] [mlir][TD] Allow op printing flags as `transform.print` attrs (#86846) Introduce 3 new optional attributes to the `transform.print` ops: * `assume_verified` * `use_local_scope` * `skip_regions` The primary motivation is to allow printing on large inputs that otherwise take forever to print and verify. For the full context, see this IREE issue: https://github.com/openxla/iree/issues/16901. Also add some tests and fix the op description. --- .../mlir/Dialect/Transform/IR/TransformOps.td | 19 ++++++- .../lib/Dialect/Transform/IR/TransformOps.cpp | 19 ++++++- mlir/test/Dialect/Transform/ops.mlir | 10 ++-- .../Transform/test-interpreter-printing.mlir | 56 +++++++++++++++++++ 4 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 mlir/test/Dialect/Transform/test-interpreter-printing.mlir diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td index bf1a8016cd9d..21c9595860d4 100644 --- a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td +++ b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td @@ -1098,15 +1098,28 @@ def PrintOp : TransformDialectOp<"print", MatchOpInterface]> { let summary = "Dump each payload op"; let description = [{ - This op dumps each payload op that is associated with the `target` operand - to stderr. It also prints the `name` string attribute. If no target is + Prints each payload op that is associated with the `target` operand to + `stdout`. It also prints the `name` string attribute. If no target is specified, the top-level op is dumped. This op is useful for printf-style debugging. + + Supported printing flag attributes: + * `assume_verified` -- skips verification when the unit attribute is + specified. This improves performace but may lead to crashes and + unexpected behavior when the printed payload op is invalid. + * `use_local_scope` -- prints in local scope when the unit attribute is + specified. This improves performance but may not be identical to + printing within the full module. + * `skip_regions` -- does not print regions of operations when the unit + attribute is specified. }]; let arguments = (ins Optional:$target, - OptionalAttr:$name); + OptionalAttr:$name, + OptionalAttr:$assume_verified, + OptionalAttr:$use_local_scope, + OptionalAttr:$skip_regions); let results = (outs); let builders = [ diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp index c8d06ba157b9..dc19022219e5 100644 --- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp +++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp @@ -19,6 +19,7 @@ #include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/Diagnostics.h" #include "mlir/IR/Dominance.h" +#include "mlir/IR/OperationSupport.h" #include "mlir/IR/PatternMatch.h" #include "mlir/IR/Verifier.h" #include "mlir/Interfaces/CallInterfaces.h" @@ -2627,14 +2628,26 @@ transform::PrintOp::apply(transform::TransformRewriter &rewriter, if (getName().has_value()) llvm::outs() << *getName() << " "; + OpPrintingFlags printFlags; + if (getAssumeVerified().value_or(false)) + printFlags.assumeVerified(); + if (getUseLocalScope().value_or(false)) + printFlags.useLocalScope(); + if (getSkipRegions().value_or(false)) + printFlags.skipRegions(); + if (!getTarget()) { - llvm::outs() << "top-level ]]]\n" << *state.getTopLevel() << "\n"; + llvm::outs() << "top-level ]]]\n"; + state.getTopLevel()->print(llvm::outs(), printFlags); + llvm::outs() << "\n"; return DiagnosedSilenceableFailure::success(); } llvm::outs() << "]]]\n"; - for (Operation *target : state.getPayloadOps(getTarget())) - llvm::outs() << *target << "\n"; + for (Operation *target : state.getPayloadOps(getTarget())) { + target->print(llvm::outs(), printFlags); + llvm::outs() << "\n"; + } return DiagnosedSilenceableFailure::success(); } diff --git a/mlir/test/Dialect/Transform/ops.mlir b/mlir/test/Dialect/Transform/ops.mlir index a718d6a9e9fd..ecef7e181e90 100644 --- a/mlir/test/Dialect/Transform/ops.mlir +++ b/mlir/test/Dialect/Transform/ops.mlir @@ -86,16 +86,18 @@ transform.sequence failures(propagate) { } // CHECK: transform.sequence -// CHECK: print -// CHECK: print -// CHECK: print -// CHECK: print +// CHECK-COUNT-9: print transform.sequence failures(propagate) { ^bb0(%arg0: !transform.any_op): transform.print %arg0 : !transform.any_op transform.print transform.print %arg0 {name = "test"} : !transform.any_op transform.print {name = "test"} + transform.print {name = "test", assume_verified} + transform.print %arg0 {assume_verified} : !transform.any_op + transform.print %arg0 {use_local_scope} : !transform.any_op + transform.print %arg0 {skip_regions} : !transform.any_op + transform.print %arg0 {assume_verified, use_local_scope, skip_regions} : !transform.any_op } // CHECK: transform.sequence diff --git a/mlir/test/Dialect/Transform/test-interpreter-printing.mlir b/mlir/test/Dialect/Transform/test-interpreter-printing.mlir new file mode 100644 index 000000000000..a54c83d2b249 --- /dev/null +++ b/mlir/test/Dialect/Transform/test-interpreter-printing.mlir @@ -0,0 +1,56 @@ +// RUN: mlir-opt %s --transform-interpreter --allow-unregistered-dialect --verify-diagnostics | FileCheck %s + +// RUN: mlir-opt %s --transform-interpreter --allow-unregistered-dialect --verify-diagnostics \ +// RUN: --mlir-print-debuginfo | FileCheck %s --check-prefix=CHECK-LOC + +func.func @nested_ops() { + "test.qux"() ({ + // expected-error @below{{fail_to_verify is set}} + "test.baz"() ({ + "test.bar"() : () -> () + }) : () -> () + }) : () -> () +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // CHECK-LABEL{LITERAL}: [[[ IR printer: START top-level ]]] + // CHECK-NEXT: module { + // CHECK-LOC-LABEL{LITERAL}: [[[ IR printer: START top-level ]]] + // CHECK-LOC-NEXT: #{{.+}} = loc( + // CHECK-LOC-NEXT: module { + transform.print {name = "START"} + + // CHECK{LITERAL}: [[[ IR printer: Local scope top-level ]]] + // CHECK-NEXT: module { + // CHECK-LOC{LITERAL}: [[[ IR printer: Local scope top-level ]]] + // CHECK-LOC-NEXT: module { + transform.print {name = "Local scope", use_local_scope} + + %baz = transform.structured.match ops{["test.baz"]} in %arg0 : (!transform.any_op) -> !transform.any_op + + // CHECK{LITERAL}: [[[ IR printer: ]]] + // CHECK-NEXT: "test.baz"() ({ + // CHECK-NEXT: "test.bar"() : () -> () + // CHECK-NEXT: }) : () -> () + transform.print %baz : !transform.any_op + + // CHECK{LITERAL}: [[[ IR printer: Baz ]]] + // CHECK-NEXT: "test.baz"() ({ + transform.print %baz {name = "Baz"} : !transform.any_op + + // CHECK{LITERAL}: [[[ IR printer: No region ]]] + // CHECK-NEXT: "test.baz"() ({...}) : () -> () + transform.print %baz {name = "No region", skip_regions} : !transform.any_op + + // CHECK{LITERAL}: [[[ IR printer: No verify ]]] + // CHECK-NEXT: "test.baz"() ({ + // CHECK-NEXT: transform.test_dummy_payload_op {fail_to_verify} : () -> () + transform.test_produce_invalid_ir %baz : !transform.any_op + transform.print %baz {name = "No verify", assume_verified} : !transform.any_op + + // CHECK-LABEL{LITERAL}: [[[ IR printer: END top-level ]]] + transform.print {name = "END"} + transform.yield + } +} -- GitLab From d83271b093ec206c2f47a9c636a5727cf63cad5e Mon Sep 17 00:00:00 2001 From: Caslyn Tonelli <6718161+Caslyn@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:51:38 -0700 Subject: [PATCH 0117/1690] [libc] Include algorithm.h to parser.h (#87125) This includes algorithm.h directly to provide the definition for `cpp:max` in parser.h. This will define `max(...)` in the libc namespace for build systems that pull in parser.h explicitly. --- libc/src/stdio/printf_core/CMakeLists.txt | 1 + libc/src/stdio/printf_core/parser.h | 1 + utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 1 + 3 files changed, 3 insertions(+) diff --git a/libc/src/stdio/printf_core/CMakeLists.txt b/libc/src/stdio/printf_core/CMakeLists.txt index 02819ea25ea0..7db79c54beb0 100644 --- a/libc/src/stdio/printf_core/CMakeLists.txt +++ b/libc/src/stdio/printf_core/CMakeLists.txt @@ -42,6 +42,7 @@ add_header_library( libc.src.__support.arg_list libc.src.__support.ctype_utils libc.src.__support.str_to_integer + libc.src.__support.CPP.algorithm libc.src.__support.CPP.bit libc.src.__support.CPP.optional libc.src.__support.CPP.string_view diff --git a/libc/src/stdio/printf_core/parser.h b/libc/src/stdio/printf_core/parser.h index 8e8c77e219fa..eda978a83ea8 100644 --- a/libc/src/stdio/printf_core/parser.h +++ b/libc/src/stdio/printf_core/parser.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PARSER_H #include "include/llvm-libc-macros/stdfix-macros.h" +#include "src/__support/CPP/algorithm.h" // max #include "src/__support/CPP/optional.h" #include "src/__support/CPP/type_traits.h" #include "src/__support/str_to_integer.h" diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 9169f330baac..9dfe4c48184e 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -3041,6 +3041,7 @@ libc_support_library( deps = [ ":__support_arg_list", ":__support_common", + ":__support_cpp_algorithm", ":__support_cpp_bit", ":__support_cpp_optional", ":__support_cpp_string_view", -- GitLab From 5bbc640f64efb7d110559dab132c8d2fb7fbbf37 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 1 Apr 2024 09:55:24 -0700 Subject: [PATCH 0118/1690] [nfc] Disable the a cpp compiler-rt test on ppc bigendian systems due to build errors (#87262) `Linux/instrprof-vtable-value-prof.cpp` needs to be built for the test to run. However, cpp compile & link failed with undefined-ABI error [1]. See original failure in https://lab.llvm.org/buildbot/#/builders/18/builds/16429 [1] ``` FAIL: Profile-powerpc64 :: Linux/instrprof-vtable-value-prof.cpp (2406 of 2414) ******************** TEST 'Profile-powerpc64 :: Linux/instrprof-vtable-value-prof.cpp' FAILED ******************** Exit Code: 1 Command Output (stderr): -- RUN: at line 3: /home/buildbots/llvm-external-buildbots/workers/ppc64be-sanitizer/sanitizer-ppc64be/build/build_debug/./bin/clang --driver-mode=g++ -m64 -ldl -fprofile-generate -fuse-ld=lld -O2 -g -fprofile-generate=. -mllvm -enable-vtable-value-profiling /home/buildbots/llvm-external-buildbots/workers/ppc64be-sanitizer/sanitizer-ppc64be/build/llvm-project/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp -o /home/buildbots/llvm-external-buildbots/workers/ppc64be-sanitizer/sanitizer-ppc64be/build/build_debug/runtimes/runtimes-bins/compiler-rt/test/profile/Profile-powerpc64/Linux/Output/instrprof-vtable-value-prof.cpp.tmp-test + /home/buildbots/llvm-external-buildbots/workers/ppc64be-sanitizer/sanitizer-ppc64be/build/build_debug/./bin/clang --driver-mode=g++ -m64 -ldl -fprofile-generate -fuse-ld=lld -O2 -g -fprofile-generate=. -mllvm -enable-vtable-value-profiling /home/buildbots/llvm-external-buildbots/workers/ppc64be-sanitizer/sanitizer-ppc64be/build/llvm-project/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp -o /home/buildbots/llvm-external-buildbots/workers/ppc64be-sanitizer/sanitizer-ppc64be/build/build_debug/runtimes/runtimes-bins/compiler-rt/test/profile/Profile-powerpc64/Linux/Output/instrprof-vtable-value-prof.cpp.tmp-test ld.lld: error: /lib/../lib64/Scrt1.o: ABI version 1 is not supported clang: error: linker command failed with exit code 1 (use -v to see invocation) ``` --- .../test/profile/Linux/instrprof-vtable-value-prof.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp index 5c8426b40892..e51805bdf923 100644 --- a/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp +++ b/compiler-rt/test/profile/Linux/instrprof-vtable-value-prof.cpp @@ -1,5 +1,10 @@ // REQUIRES: lld-available +// Building the instrumented binary will fail because lld doesn't support +// big-endian ELF for PPC (aka ABI 1). +// ld.lld: error: /lib/../lib64/Scrt1.o: ABI version 1 is not supported +// UNSUPPORTED: ppc && host-byteorder-big-endian + // RUN: %clangxx_pgogen -fuse-ld=lld -O2 -g -fprofile-generate=. -mllvm -enable-vtable-value-profiling %s -o %t-test // RUN: env LLVM_PROFILE_FILE=%t-test.profraw %t-test -- GitLab From 55b74030a4c75f25be901522fe595d7233fad76d Mon Sep 17 00:00:00 2001 From: Shourya Goel Date: Mon, 1 Apr 2024 22:39:28 +0530 Subject: [PATCH 0119/1690] [libc][POSIX] implement fseeko, ftello (#86928) Fixes: #85287 --- libc/config/linux/x86_64/entrypoints.txt | 2 ++ libc/src/stdio/CMakeLists.txt | 2 ++ libc/src/stdio/fseeko.h | 20 ++++++++++++++++++ libc/src/stdio/ftello.h | 20 ++++++++++++++++++ libc/src/stdio/generic/CMakeLists.txt | 26 ++++++++++++++++++++++-- libc/src/stdio/generic/fseek.cpp | 1 - libc/src/stdio/generic/fseeko.cpp | 26 ++++++++++++++++++++++++ libc/src/stdio/generic/ftell.cpp | 1 - libc/src/stdio/generic/ftello.cpp | 25 +++++++++++++++++++++++ libc/test/src/stdio/CMakeLists.txt | 2 ++ libc/test/src/stdio/ftell_test.cpp | 9 ++++++++ 11 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 libc/src/stdio/fseeko.h create mode 100644 libc/src/stdio/ftello.h create mode 100644 libc/src/stdio/generic/fseeko.cpp create mode 100644 libc/src/stdio/generic/ftello.cpp diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 5b428e51aee6..a4d0da5e043d 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -681,6 +681,8 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.stdio.fread_unlocked libc.src.stdio.fseek libc.src.stdio.ftell + libc.src.stdio.fseeko + libc.src.stdio.ftello libc.src.stdio.funlockfile libc.src.stdio.fwrite libc.src.stdio.fwrite_unlocked diff --git a/libc/src/stdio/CMakeLists.txt b/libc/src/stdio/CMakeLists.txt index 11e15c917351..1056f38fc751 100644 --- a/libc/src/stdio/CMakeLists.txt +++ b/libc/src/stdio/CMakeLists.txt @@ -270,6 +270,8 @@ add_stdio_entrypoint_object(ferror) add_stdio_entrypoint_object(ferror_unlocked) add_stdio_entrypoint_object(fseek) add_stdio_entrypoint_object(ftell) +add_stdio_entrypoint_object(fseeko) +add_stdio_entrypoint_object(ftello) add_stdio_entrypoint_object(fflush) add_stdio_entrypoint_object(clearerr) add_stdio_entrypoint_object(clearerr_unlocked) diff --git a/libc/src/stdio/fseeko.h b/libc/src/stdio/fseeko.h new file mode 100644 index 000000000000..77fb41215c31 --- /dev/null +++ b/libc/src/stdio/fseeko.h @@ -0,0 +1,20 @@ +//===-- Implementation header of fseeko -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDIO_FSEEKO_H +#define LLVM_LIBC_SRC_STDIO_FSEEKO_H + +#include + +namespace LIBC_NAMESPACE { + +int fseeko(::FILE *stream, off_t offset, int whence); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDIO_FSEEKO_H diff --git a/libc/src/stdio/ftello.h b/libc/src/stdio/ftello.h new file mode 100644 index 000000000000..5ab17f9244a5 --- /dev/null +++ b/libc/src/stdio/ftello.h @@ -0,0 +1,20 @@ +//===-- Implementation header of ftello -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDIO_FTELLO_H +#define LLVM_LIBC_SRC_STDIO_FTELLO_H + +#include + +namespace LIBC_NAMESPACE { + +off_t ftello(::FILE *f); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDIO_FTELLO_H diff --git a/libc/src/stdio/generic/CMakeLists.txt b/libc/src/stdio/generic/CMakeLists.txt index 0aa213caba7b..ae255917adfe 100644 --- a/libc/src/stdio/generic/CMakeLists.txt +++ b/libc/src/stdio/generic/CMakeLists.txt @@ -103,7 +103,6 @@ add_entrypoint_object( ../fseek.h DEPENDS libc.src.errno.errno - libc.include.stdio libc.src.__support.File.file libc.src.__support.File.platform_file ) @@ -116,7 +115,30 @@ add_entrypoint_object( ../ftell.h DEPENDS libc.src.errno.errno - libc.include.stdio + libc.src.__support.File.file + libc.src.__support.File.platform_file +) + +add_entrypoint_object( + fseeko + SRCS + fseeko.cpp + HDRS + ../fseeko.h + DEPENDS + libc.src.errno.errno + libc.src.__support.File.file + libc.src.__support.File.platform_file +) + +add_entrypoint_object( + ftello + SRCS + ftello.cpp + HDRS + ../ftello.h + DEPENDS + libc.src.errno.errno libc.src.__support.File.file libc.src.__support.File.platform_file ) diff --git a/libc/src/stdio/generic/fseek.cpp b/libc/src/stdio/generic/fseek.cpp index 7666e71e699d..c5edc8d4198c 100644 --- a/libc/src/stdio/generic/fseek.cpp +++ b/libc/src/stdio/generic/fseek.cpp @@ -10,7 +10,6 @@ #include "src/__support/File/file.h" #include "src/errno/libc_errno.h" -#include namespace LIBC_NAMESPACE { diff --git a/libc/src/stdio/generic/fseeko.cpp b/libc/src/stdio/generic/fseeko.cpp new file mode 100644 index 000000000000..215da759937f --- /dev/null +++ b/libc/src/stdio/generic/fseeko.cpp @@ -0,0 +1,26 @@ +//===-- Implementation of fseeko ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdio/fseeko.h" +#include "src/__support/File/file.h" + +#include "src/errno/libc_errno.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, fseeko, (::FILE * stream, off_t offset, int whence)) { + auto result = + reinterpret_cast(stream)->seek(offset, whence); + if (!result.has_value()) { + libc_errno = result.error(); + return -1; + } + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdio/generic/ftell.cpp b/libc/src/stdio/generic/ftell.cpp index 5f7803150534..d55bad282854 100644 --- a/libc/src/stdio/generic/ftell.cpp +++ b/libc/src/stdio/generic/ftell.cpp @@ -10,7 +10,6 @@ #include "src/__support/File/file.h" #include "src/errno/libc_errno.h" -#include namespace LIBC_NAMESPACE { diff --git a/libc/src/stdio/generic/ftello.cpp b/libc/src/stdio/generic/ftello.cpp new file mode 100644 index 000000000000..c72e56ea6eb1 --- /dev/null +++ b/libc/src/stdio/generic/ftello.cpp @@ -0,0 +1,25 @@ +//===-- Implementation of ftello ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdio/ftello.h" +#include "src/__support/File/file.h" + +#include "src/errno/libc_errno.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(off_t, ftello, (::FILE * stream)) { + auto result = reinterpret_cast(stream)->tell(); + if (!result.has_value()) { + libc_errno = result.error(); + return -1; + } + return result.value(); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/test/src/stdio/CMakeLists.txt b/libc/test/src/stdio/CMakeLists.txt index 4c38e8aba7d7..03c43eaefebe 100644 --- a/libc/test/src/stdio/CMakeLists.txt +++ b/libc/test/src/stdio/CMakeLists.txt @@ -442,6 +442,8 @@ add_libc_test( libc.src.stdio.fread libc.src.stdio.fseek libc.src.stdio.ftell + libc.src.stdio.fseeko + libc.src.stdio.ftello libc.src.stdio.fwrite libc.src.stdio.setvbuf ) diff --git a/libc/test/src/stdio/ftell_test.cpp b/libc/test/src/stdio/ftell_test.cpp index 61b626f53cd2..68a969ed0c30 100644 --- a/libc/test/src/stdio/ftell_test.cpp +++ b/libc/test/src/stdio/ftell_test.cpp @@ -10,7 +10,9 @@ #include "src/stdio/fopen.h" #include "src/stdio/fread.h" #include "src/stdio/fseek.h" +#include "src/stdio/fseeko.h" #include "src/stdio/ftell.h" +#include "src/stdio/ftello.h" #include "src/stdio/fwrite.h" #include "src/stdio/setvbuf.h" #include "test/UnitTest/Test.h" @@ -37,6 +39,13 @@ protected: // still return the correct effective offset. ASSERT_EQ(size_t(LIBC_NAMESPACE::ftell(file)), WRITE_SIZE); + off_t offseto = 42; + ASSERT_EQ(0, LIBC_NAMESPACE::fseeko(file, offseto, SEEK_SET)); + ASSERT_EQ(LIBC_NAMESPACE::ftello(file), offseto); + ASSERT_EQ(0, LIBC_NAMESPACE::fseeko(file, -offseto, SEEK_END)); + ASSERT_EQ(size_t(LIBC_NAMESPACE::ftello(file)), + size_t(WRITE_SIZE - offseto)); + long offset = 5; ASSERT_EQ(0, LIBC_NAMESPACE::fseek(file, offset, SEEK_SET)); ASSERT_EQ(LIBC_NAMESPACE::ftell(file), offset); -- GitLab From 92d0d6f6cb4099e651d066cd88dc0abfa6e612cf Mon Sep 17 00:00:00 2001 From: ChiaHungDuan Date: Mon, 1 Apr 2024 10:09:40 -0700 Subject: [PATCH 0120/1690] [scudo] Do a M_PURGE call before changing release interval on Android (#87110) --- compiler-rt/lib/scudo/standalone/wrappers_c.inc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.inc b/compiler-rt/lib/scudo/standalone/wrappers_c.inc index 56d8ef20156e..21d5b7add512 100644 --- a/compiler-rt/lib/scudo/standalone/wrappers_c.inc +++ b/compiler-rt/lib/scudo/standalone/wrappers_c.inc @@ -247,6 +247,11 @@ void SCUDO_PREFIX(malloc_postinit)() { INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, int value) { if (param == M_DECAY_TIME) { if (SCUDO_ANDROID) { + // Before changing the interval, reset the memory usage status by doing a + // M_PURGE call so that we can minimize the impact of any unreleased pages + // introduced by interval transition. + SCUDO_ALLOCATOR.releaseToOS(scudo::ReleaseToOS::Force); + if (value == 0) { // Will set the release values to their minimum values. value = INT32_MIN; -- GitLab From 9434c083475e42f47383f3067fe2a155db5c6a30 Mon Sep 17 00:00:00 2001 From: Chris B Date: Mon, 1 Apr 2024 12:10:10 -0500 Subject: [PATCH 0121/1690] [HLSL] Implement array temporary support (#79382) HLSL constant sized array function parameters do not decay to pointers. Instead constant sized array types are preserved as unique types for overload resolution, template instantiation and name mangling. This implements the change by adding a new `ArrayParameterType` which represents a non-decaying `ConstantArrayType`. The new type behaves the same as `ConstantArrayType` except that it does not decay to a pointer. Values of `ConstantArrayType` in HLSL decay during overload resolution via a new `HLSLArrayRValue` cast to `ArrayParameterType`. `ArrayParamterType` values are passed indirectly by-value to functions in IR generation resulting in callee generated memcpy instructions. The behavior of HLSL function calls is documented in the [draft language specification](https://microsoft.github.io/hlsl-specs/specs/hlsl.pdf) under the Expr.Post.Call heading. Additionally the design of this implementation approach is documented in [Clang's documentation](https://clang.llvm.org/docs/HLSL/FunctionCalls.html) Resolves #70123 --- clang/docs/HLSL/FunctionCalls.rst | 31 +++--- clang/include/clang/AST/ASTContext.h | 7 ++ clang/include/clang/AST/OperationKinds.def | 3 + clang/include/clang/AST/RecursiveASTVisitor.h | 11 ++ clang/include/clang/AST/Type.h | 45 +++++++- clang/include/clang/AST/TypeLoc.h | 5 + clang/include/clang/AST/TypeProperties.td | 7 ++ clang/include/clang/Basic/TypeNodes.td | 1 + clang/include/clang/Sema/Overload.h | 3 + .../clang/Serialization/TypeBitCodes.def | 1 + clang/lib/AST/ASTContext.cpp | 62 ++++++++++- clang/lib/AST/ASTImporter.cpp | 9 ++ clang/lib/AST/ASTStructuralEquivalence.cpp | 1 + clang/lib/AST/Expr.cpp | 1 + clang/lib/AST/ExprConstant.cpp | 3 + clang/lib/AST/ItaniumMangle.cpp | 5 + clang/lib/AST/MicrosoftMangle.cpp | 5 + clang/lib/AST/ODRHash.cpp | 4 + clang/lib/AST/Type.cpp | 11 ++ clang/lib/AST/TypePrinter.cpp | 11 ++ clang/lib/CodeGen/CGCall.cpp | 3 +- clang/lib/CodeGen/CGDebugInfo.cpp | 1 + clang/lib/CodeGen/CGExpr.cpp | 1 + clang/lib/CodeGen/CGExprAgg.cpp | 4 + clang/lib/CodeGen/CGExprComplex.cpp | 1 + clang/lib/CodeGen/CGExprConstant.cpp | 1 + clang/lib/CodeGen/CGExprScalar.cpp | 1 + clang/lib/CodeGen/CodeGenFunction.cpp | 2 + clang/lib/CodeGen/CodeGenTypes.cpp | 1 + clang/lib/CodeGen/ItaniumCXXABI.cpp | 4 + clang/lib/Edit/RewriteObjCFoundationAPI.cpp | 1 + clang/lib/Sema/Sema.cpp | 1 + clang/lib/Sema/SemaExpr.cpp | 10 +- clang/lib/Sema/SemaExprCXX.cpp | 8 ++ clang/lib/Sema/SemaInit.cpp | 5 +- clang/lib/Sema/SemaLookup.cpp | 4 + clang/lib/Sema/SemaOverload.cpp | 19 +++- clang/lib/Sema/SemaTemplate.cpp | 5 + clang/lib/Sema/SemaTemplateDeduction.cpp | 3 +- clang/lib/Sema/SemaType.cpp | 3 + clang/lib/Sema/TreeTransform.h | 17 +++ clang/lib/Serialization/ASTReader.cpp | 4 + clang/lib/Serialization/ASTWriter.cpp | 4 + clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp | 3 +- clang/test/CodeGenHLSL/ArrayTemporary.hlsl | 104 ++++++++++++++++++ clang/test/SemaHLSL/ArrayParams.hlsl | 29 +++++ clang/test/SemaHLSL/ArrayTemporary.hlsl | 95 ++++++++++++++++ clang/test/SemaHLSL/ArrayTemporary.ll | 76 +++++++++++++ clang/tools/libclang/CIndex.cpp | 1 + 49 files changed, 606 insertions(+), 31 deletions(-) create mode 100644 clang/test/CodeGenHLSL/ArrayTemporary.hlsl create mode 100644 clang/test/SemaHLSL/ArrayParams.hlsl create mode 100644 clang/test/SemaHLSL/ArrayTemporary.hlsl create mode 100644 clang/test/SemaHLSL/ArrayTemporary.ll diff --git a/clang/docs/HLSL/FunctionCalls.rst b/clang/docs/HLSL/FunctionCalls.rst index 7317de2163f8..6d65fe6e3fb2 100644 --- a/clang/docs/HLSL/FunctionCalls.rst +++ b/clang/docs/HLSL/FunctionCalls.rst @@ -157,22 +157,23 @@ Clang Implementation of the changes in the prototype implementation are restoring Clang-3.7 code that was previously modified to its original state. -The implementation in clang depends on two new AST nodes and minor extensions to -Clang's existing support for Objective-C write-back arguments. The goal of this -design is to capture the semantic details of HLSL function calls in the AST, and -minimize the amount of magic that needs to occur during IR generation. - -The two new AST nodes are ``HLSLArrayTemporaryExpr`` and ``HLSLOutParamExpr``, -which respectively represent the temporaries used for passing arrays by value -and the temporaries created for function outputs. +The implementation in clang adds a new non-decaying array type, a new AST node +to represent output parameters, and minor extensions to Clang's existing support +for Objective-C write-back arguments. The goal of this design is to capture the +semantic details of HLSL function calls in the AST, and minimize the amount of +magic that needs to occur during IR generation. Array Temporaries ----------------- -The ``HLSLArrayTemporaryExpr`` represents temporary values for input -constant-sized array arguments. This applies for all constant-sized array -arguments regardless of whether or not the parameter is constant-sized or -unsized. +The new ``ArrayParameterType`` is a sub-class of ``ConstantArrayType`` +inheriting all the behaviors and methods of the parent except that it does not +decay to a pointer during overload resolution or template type deduction. + +An argument of ``ConstantArrayType`` can be implicitly converted to an +equivalent non-decayed ``ArrayParameterType`` if the underlying canonical +``ConstantArrayType`` is the same. This occurs during overload resolution +instead of array to pointer decay. .. code-block:: c++ @@ -193,7 +194,7 @@ In the example above, the following AST is generated for the call to CallExpr 'void' |-ImplicitCastExpr 'void (*)(float [4])' | `-DeclRefExpr 'void (float [4])' lvalue Function 'SizedArray' 'void (float [4])' - `-HLSLArrayTemporaryExpr 'float [4]' + `-ImplicitCastExpr 'float [4]' `-DeclRefExpr 'float [4]' lvalue Var 'arr' 'float [4]' In the example above, the following AST is generated for the call to @@ -204,7 +205,7 @@ In the example above, the following AST is generated for the call to CallExpr 'void' |-ImplicitCastExpr 'void (*)(float [])' | `-DeclRefExpr 'void (float [])' lvalue Function 'UnsizedArray' 'void (float [])' - `-HLSLArrayTemporaryExpr 'float [4]' + `-ImplicitCastExpr 'float [4]' `-DeclRefExpr 'float [4]' lvalue Var 'arr' 'float [4]' In both of these cases the argument expression is of known array size so we can @@ -236,7 +237,7 @@ An expected AST should be something like: CallExpr 'void' |-ImplicitCastExpr 'void (*)(float [])' | `-DeclRefExpr 'void (float [])' lvalue Function 'UnsizedArray' 'void (float [])' - `-HLSLArrayTemporaryExpr 'float [4]' + `-ImplicitCastExpr 'float [4]' `-DeclRefExpr 'float [4]' lvalue Var 'arr' 'float [4]' Out Parameter Temporaries diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h index 002f36ecbbaa..08f71051e6cb 100644 --- a/clang/include/clang/AST/ASTContext.h +++ b/clang/include/clang/AST/ASTContext.h @@ -260,6 +260,9 @@ class ASTContext : public RefCountedBase { ASTContext&> SubstTemplateTemplateParmPacks; + mutable llvm::ContextualFoldingSet + ArrayParameterTypes; + /// The set of nested name specifiers. /// /// This set is managed by the NestedNameSpecifier class. @@ -1367,6 +1370,10 @@ public: /// type to the decayed type. QualType getDecayedType(QualType Orig, QualType Decayed) const; + /// Return the uniqued reference to a specified array parameter type from the + /// original array type. + QualType getArrayParameterType(QualType Ty) const; + /// Return the uniqued reference to the atomic type for the specified /// type. QualType getAtomicType(QualType T) const; diff --git a/clang/include/clang/AST/OperationKinds.def b/clang/include/clang/AST/OperationKinds.def index ef05072800f1..8788b8ff0ef0 100644 --- a/clang/include/clang/AST/OperationKinds.def +++ b/clang/include/clang/AST/OperationKinds.def @@ -364,6 +364,9 @@ CAST_OPERATION(IntToOCLSampler) // Truncate a vector type by dropping elements from the end (HLSL only). CAST_OPERATION(HLSLVectorTruncation) +// Non-decaying array RValue cast (HLSL only). +CAST_OPERATION(HLSLArrayRValue) + //===- Binary Operations -------------------------------------------------===// // Operators listed in order of precedence. // Note that additions to this should also update the StmtVisitor class, diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h index 4a1ff222ecad..8630317795a9 100644 --- a/clang/include/clang/AST/RecursiveASTVisitor.h +++ b/clang/include/clang/AST/RecursiveASTVisitor.h @@ -993,6 +993,12 @@ DEF_TRAVERSE_TYPE(ConstantArrayType, { TRY_TO(TraverseStmt(const_cast(T->getSizeExpr()))); }) +DEF_TRAVERSE_TYPE(ArrayParameterType, { + TRY_TO(TraverseType(T->getElementType())); + if (T->getSizeExpr()) + TRY_TO(TraverseStmt(const_cast(T->getSizeExpr()))); +}) + DEF_TRAVERSE_TYPE(IncompleteArrayType, { TRY_TO(TraverseType(T->getElementType())); }) @@ -1260,6 +1266,11 @@ DEF_TRAVERSE_TYPELOC(ConstantArrayType, { TRY_TO(TraverseArrayTypeLocHelper(TL)); }) +DEF_TRAVERSE_TYPELOC(ArrayParameterType, { + TRY_TO(TraverseTypeLoc(TL.getElementLoc())); + TRY_TO(TraverseArrayTypeLocHelper(TL)); +}) + DEF_TRAVERSE_TYPELOC(IncompleteArrayType, { TRY_TO(TraverseTypeLoc(TL.getElementLoc())); TRY_TO(TraverseArrayTypeLocHelper(TL)); diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h index 5d8dde37e769..99f45d518c79 100644 --- a/clang/include/clang/AST/Type.h +++ b/clang/include/clang/AST/Type.h @@ -2300,6 +2300,7 @@ public: bool isConstantArrayType() const; bool isIncompleteArrayType() const; bool isVariableArrayType() const; + bool isArrayParameterType() const; bool isDependentSizedArrayType() const; bool isRecordType() const; bool isClassType() const; @@ -3334,14 +3335,15 @@ public: return T->getTypeClass() == ConstantArray || T->getTypeClass() == VariableArray || T->getTypeClass() == IncompleteArray || - T->getTypeClass() == DependentSizedArray; + T->getTypeClass() == DependentSizedArray || + T->getTypeClass() == ArrayParameter; } }; /// Represents the canonical version of C arrays with a specified constant size. /// For example, the canonical type for 'int A[4 + 4*100]' is a /// ConstantArrayType where the element type is 'int' and the size is 404. -class ConstantArrayType final : public ArrayType { +class ConstantArrayType : public ArrayType { friend class ASTContext; // ASTContext creates these. struct ExternalSize { @@ -3382,6 +3384,19 @@ class ConstantArrayType final : public ArrayType { const Expr *SzExpr, ArraySizeModifier SzMod, unsigned Qual); +protected: + ConstantArrayType(TypeClass Tc, const ConstantArrayType *ATy, QualType Can) + : ArrayType(Tc, ATy->getElementType(), Can, ATy->getSizeModifier(), + ATy->getIndexTypeQualifiers().getAsOpaqueValue(), nullptr) { + ConstantArrayTypeBits.HasExternalSize = + ATy->ConstantArrayTypeBits.HasExternalSize; + if (!ConstantArrayTypeBits.HasExternalSize) { + ConstantArrayTypeBits.SizeWidth = ATy->ConstantArrayTypeBits.SizeWidth; + Size = ATy->Size; + } else + SizePtr = ATy->SizePtr; + } + public: /// Return the constant array size as an APInt. llvm::APInt getSize() const { @@ -3453,7 +3468,22 @@ public: ArraySizeModifier SizeMod, unsigned TypeQuals); static bool classof(const Type *T) { - return T->getTypeClass() == ConstantArray; + return T->getTypeClass() == ConstantArray || + T->getTypeClass() == ArrayParameter; + } +}; + +/// Represents a constant array type that does not decay to a pointer when used +/// as a function parameter. +class ArrayParameterType : public ConstantArrayType { + friend class ASTContext; // ASTContext creates these. + + ArrayParameterType(const ConstantArrayType *ATy, QualType CanTy) + : ConstantArrayType(ArrayParameter, ATy, CanTy) {} + +public: + static bool classof(const Type *T) { + return T->getTypeClass() == ArrayParameter; } }; @@ -7185,7 +7215,8 @@ inline bool QualType::isCanonicalAsParam() const { if (T->isVariablyModifiedType() && T->hasSizedVLAType()) return false; - return !isa(T) && !isa(T); + return !isa(T) && + (!isa(T) || isa(T)); } inline bool QualType::isConstQualified() const { @@ -7450,6 +7481,10 @@ inline bool Type::isVariableArrayType() const { return isa(CanonicalType); } +inline bool Type::isArrayParameterType() const { + return isa(CanonicalType); +} + inline bool Type::isDependentSizedArrayType() const { return isa(CanonicalType); } @@ -7813,7 +7848,7 @@ inline bool Type::isTypedefNameType() const { /// Determines whether this type can decay to a pointer type. inline bool Type::canDecayToPointerType() const { - return isFunctionType() || isArrayType(); + return isFunctionType() || (isArrayType() && !isArrayParameterType()); } inline bool Type::hasPointerRepresentation() const { diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h index b09eb3539a4b..9f2dff7a782c 100644 --- a/clang/include/clang/AST/TypeLoc.h +++ b/clang/include/clang/AST/TypeLoc.h @@ -1611,6 +1611,11 @@ class ConstantArrayTypeLoc : ConstantArrayType> { }; +/// Wrapper for source info for array parameter types. +class ArrayParameterTypeLoc + : public InheritingConcreteTypeLoc< + ConstantArrayTypeLoc, ArrayParameterTypeLoc, ArrayParameterType> {}; + class IncompleteArrayTypeLoc : public InheritingConcreteTypeLoc; } +let Class = ArrayParameterType in { + def : Creator<[{ return ctx.getAdjustedParameterType( + ctx.getConstantArrayType(elementType,sizeValue, + size,sizeModifier, + indexQualifiers.getCVRQualifiers())); }]>; +} + let Class = IncompleteArrayType in { def : Creator<[{ return ctx.getIncompleteArrayType(elementType, sizeModifier, diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td index 3625f0637589..fee49cf4326d 100644 --- a/clang/include/clang/Basic/TypeNodes.td +++ b/clang/include/clang/Basic/TypeNodes.td @@ -64,6 +64,7 @@ def ConstantArrayType : TypeNode; def IncompleteArrayType : TypeNode; def VariableArrayType : TypeNode; def DependentSizedArrayType : TypeNode, AlwaysDependent; +def ArrayParameterType : TypeNode; def DependentSizedExtVectorType : TypeNode, AlwaysDependent; def DependentAddressSpaceType : TypeNode, AlwaysDependent; def VectorType : TypeNode; diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h index e4717dd5baf1..76311b00d2fc 100644 --- a/clang/include/clang/Sema/Overload.h +++ b/clang/include/clang/Sema/Overload.h @@ -198,6 +198,9 @@ class Sema; /// HLSL vector truncation. ICK_HLSL_Vector_Truncation, + /// HLSL non-decaying array rvalue cast. + ICK_HLSL_Array_RValue, + /// The number of conversion kinds ICK_Num_Conversion_Kinds, }; diff --git a/clang/include/clang/Serialization/TypeBitCodes.def b/clang/include/clang/Serialization/TypeBitCodes.def index 3c82dfed9497..82b053d4caca 100644 --- a/clang/include/clang/Serialization/TypeBitCodes.def +++ b/clang/include/clang/Serialization/TypeBitCodes.def @@ -66,5 +66,6 @@ TYPE_BIT_CODE(Using, USING, 54) TYPE_BIT_CODE(BTFTagAttributed, BTFTAG_ATTRIBUTED, 55) TYPE_BIT_CODE(PackIndexing, PACK_INDEXING, 56) TYPE_BIT_CODE(CountAttributed, COUNT_ATTRIBUTED, 57) +TYPE_BIT_CODE(ArrayParameter, ARRAY_PARAMETER, 58) #undef TYPE_BIT_CODE diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp index c90fafb6f653..f7f55dc4e7a9 100644 --- a/clang/lib/AST/ASTContext.cpp +++ b/clang/lib/AST/ASTContext.cpp @@ -879,7 +879,8 @@ ASTContext::ASTContext(LangOptions &LOpts, SourceManager &SM, TemplateSpecializationTypes(this_()), DependentTemplateSpecializationTypes(this_()), AutoTypes(this_()), DependentBitIntTypes(this_()), SubstTemplateTemplateParmPacks(this_()), - CanonTemplateTemplateParms(this_()), SourceMgr(SM), LangOpts(LOpts), + ArrayParameterTypes(this_()), CanonTemplateTemplateParms(this_()), + SourceMgr(SM), LangOpts(LOpts), NoSanitizeL(new NoSanitizeList(LangOpts.NoSanitizeFiles, SM)), XRayFilter(new XRayFunctionFilter(LangOpts.XRayAlwaysInstrumentFiles, LangOpts.XRayNeverInstrumentFiles, @@ -1906,7 +1907,8 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const { case Type::IncompleteArray: case Type::VariableArray: - case Type::ConstantArray: { + case Type::ConstantArray: + case Type::ArrayParameter: { // Model non-constant sized arrays as size zero, but track the alignment. uint64_t Size = 0; if (const auto *CAT = dyn_cast(T)) @@ -3396,6 +3398,37 @@ QualType ASTContext::getDecayedType(QualType T) const { return getDecayedType(T, Decayed); } +QualType ASTContext::getArrayParameterType(QualType Ty) const { + if (Ty->isArrayParameterType()) + return Ty; + assert(Ty->isConstantArrayType() && "Ty must be an array type."); + const auto *ATy = cast(Ty); + llvm::FoldingSetNodeID ID; + ATy->Profile(ID, *this, ATy->getElementType(), ATy->getZExtSize(), + ATy->getSizeExpr(), ATy->getSizeModifier(), + ATy->getIndexTypeQualifiers().getAsOpaqueValue()); + void *InsertPos = nullptr; + ArrayParameterType *AT = + ArrayParameterTypes.FindNodeOrInsertPos(ID, InsertPos); + if (AT) + return QualType(AT, 0); + + QualType Canonical; + if (!Ty.isCanonical()) { + Canonical = getArrayParameterType(getCanonicalType(Ty)); + + // Get the new insert position for the node we care about. + AT = ArrayParameterTypes.FindNodeOrInsertPos(ID, InsertPos); + assert(!AT && "Shouldn't be in the map!"); + } + + AT = new (*this, alignof(ArrayParameterType)) + ArrayParameterType(ATy, Canonical); + Types.push_back(AT); + ArrayParameterTypes.InsertNode(AT, InsertPos); + return QualType(AT, 0); +} + /// getBlockPointerType - Return the uniqued reference to the type for /// a pointer to the specified block. QualType ASTContext::getBlockPointerType(QualType T) const { @@ -3642,6 +3675,7 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const { case Type::PackIndexing: case Type::BitInt: case Type::DependentBitInt: + case Type::ArrayParameter: llvm_unreachable("type should never be variably-modified"); // These types can be variably-modified but should never need to @@ -6051,7 +6085,9 @@ CanQualType ASTContext::getCanonicalParamType(QualType T) const { T = getVariableArrayDecayedType(T); const Type *Ty = T.getTypePtr(); QualType Result; - if (isa(Ty)) { + if (getLangOpts().HLSL && isa(Ty)) { + Result = getArrayParameterType(QualType(Ty, 0)); + } else if (isa(Ty)) { Result = getArrayDecayedType(QualType(Ty,0)); } else if (isa(Ty)) { Result = getPointerType(QualType(Ty, 0)); @@ -6973,6 +7009,8 @@ const ArrayType *ASTContext::getAsArrayType(QualType T) const { } QualType ASTContext::getAdjustedParameterType(QualType T) const { + if (getLangOpts().HLSL && T->isConstantArrayType()) + return getArrayParameterType(T); if (T->isArrayType() || T->isFunctionType()) return getDecayedType(T); return T; @@ -8583,6 +8621,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S, case Type::DeducedTemplateSpecialization: return; + case Type::ArrayParameter: case Type::Pipe: #define ABSTRACT_TYPE(KIND, BASE) #define TYPE(KIND, BASE) @@ -10926,6 +10965,10 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer, assert(LHS != RHS && "Equivalent pipe types should have already been handled!"); return {}; + case Type::ArrayParameter: + assert(LHS != RHS && + "Equivalent ArrayParameter types should have already been handled!"); + return {}; case Type::BitInt: { // Merge two bit-precise int types, while trying to preserve typedef info. bool LHSUnsigned = LHS->castAs()->isUnsigned(); @@ -12817,6 +12860,18 @@ static QualType getCommonNonSugarTypeNode(ASTContext &Ctx, const Type *X, getCommonArrayElementType(Ctx, AX, QX, AY, QY), AX->getSize(), SizeExpr, getCommonSizeModifier(AX, AY), getCommonIndexTypeCVRQualifiers(AX, AY)); } + case Type::ArrayParameter: { + const auto *AX = cast(X), + *AY = cast(Y); + assert(AX->getSize() == AY->getSize()); + const Expr *SizeExpr = Ctx.hasSameExpr(AX->getSizeExpr(), AY->getSizeExpr()) + ? AX->getSizeExpr() + : nullptr; + auto ArrayTy = Ctx.getConstantArrayType( + getCommonArrayElementType(Ctx, AX, QX, AY, QY), AX->getSize(), SizeExpr, + getCommonSizeModifier(AX, AY), getCommonIndexTypeCVRQualifiers(AX, AY)); + return Ctx.getArrayParameterType(ArrayTy); + } case Type::Atomic: { const auto *AX = cast(X), *AY = cast(Y); return Ctx.getAtomicType( @@ -13078,6 +13133,7 @@ static QualType getCommonSugarTypeNode(ASTContext &Ctx, const Type *X, CANONICAL_TYPE(Builtin) CANONICAL_TYPE(Complex) CANONICAL_TYPE(ConstantArray) + CANONICAL_TYPE(ArrayParameter) CANONICAL_TYPE(ConstantMatrix) CANONICAL_TYPE(Enum) CANONICAL_TYPE(ExtVector) diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp index 786695f00fad..94a47a8f6190 100644 --- a/clang/lib/AST/ASTImporter.cpp +++ b/clang/lib/AST/ASTImporter.cpp @@ -1218,6 +1218,15 @@ ASTNodeImporter::VisitConstantArrayType(const ConstantArrayType *T) { T->getIndexTypeCVRQualifiers()); } +ExpectedType +ASTNodeImporter::VisitArrayParameterType(const ArrayParameterType *T) { + ExpectedType ToArrayTypeOrErr = VisitConstantArrayType(T); + if (!ToArrayTypeOrErr) + return ToArrayTypeOrErr.takeError(); + + return Importer.getToContext().getArrayParameterType(*ToArrayTypeOrErr); +} + ExpectedType ASTNodeImporter::VisitIncompleteArrayType(const IncompleteArrayType *T) { ExpectedType ToElementTypeOrErr = import(T->getElementType()); diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp index 226e0aa38ece..d56bf21b459e 100644 --- a/clang/lib/AST/ASTStructuralEquivalence.cpp +++ b/clang/lib/AST/ASTStructuralEquivalence.cpp @@ -840,6 +840,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context, case Type::Adjusted: case Type::Decayed: + case Type::ArrayParameter: if (!IsStructurallyEquivalent(Context, cast(T1)->getOriginalType(), cast(T2)->getOriginalType())) diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp index 6221ebd5c9b4..26efaa462405 100644 --- a/clang/lib/AST/Expr.cpp +++ b/clang/lib/AST/Expr.cpp @@ -1948,6 +1948,7 @@ bool CastExpr::CastConsistency() const { case CK_UserDefinedConversion: // operator bool() case CK_BuiltinFnToFnPtr: case CK_FixedPointToBoolean: + case CK_HLSLArrayRValue: CheckNoBasePath: assert(path_empty() && "Cast kind should not have a base path!"); break; diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index dae8f32fc029..0058e8651998 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -11699,6 +11699,7 @@ GCCTypeClass EvaluateBuiltinClassifyType(QualType T, case Type::IncompleteArray: case Type::FunctionNoProto: case Type::FunctionProto: + case Type::ArrayParameter: return GCCTypeClass::Pointer; case Type::MemberPointer: @@ -14085,6 +14086,7 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) { case CK_AtomicToNonAtomic: case CK_NoOp: case CK_LValueToRValueBitCast: + case CK_HLSLArrayRValue: return ExprEvaluatorBaseTy::VisitCastExpr(E); case CK_MemberPointerToBoolean: @@ -14913,6 +14915,7 @@ bool ComplexExprEvaluator::VisitCastExpr(const CastExpr *E) { case CK_AtomicToNonAtomic: case CK_NoOp: case CK_LValueToRValueBitCast: + case CK_HLSLArrayRValue: return ExprEvaluatorBaseTy::VisitCastExpr(E); case CK_Dependent: diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp index 425f84e8af1f..d632c697fa20 100644 --- a/clang/lib/AST/ItaniumMangle.cpp +++ b/clang/lib/AST/ItaniumMangle.cpp @@ -2398,6 +2398,7 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty, case Type::Complex: case Type::Adjusted: case Type::Decayed: + case Type::ArrayParameter: case Type::Pointer: case Type::BlockPointer: case Type::LValueReference: @@ -4446,6 +4447,10 @@ void CXXNameMangler::mangleType(const DependentBitIntType *T) { Out << "_"; } +void CXXNameMangler::mangleType(const ArrayParameterType *T) { + mangleType(cast(T)); +} + void CXXNameMangler::mangleIntegerLiteral(QualType T, const llvm::APSInt &Value) { // ::= L E # integer literal diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp index addc3140546a..a0bb04e69c9b 100644 --- a/clang/lib/AST/MicrosoftMangle.cpp +++ b/clang/lib/AST/MicrosoftMangle.cpp @@ -3079,6 +3079,11 @@ void MicrosoftCXXNameMangler::mangleArrayType(const ArrayType *T) { mangleType(ElementTy, SourceRange(), QMM_Escape); } +void MicrosoftCXXNameMangler::mangleType(const ArrayParameterType *T, + Qualifiers, SourceRange) { + mangleArrayType(cast(T)); +} + // ::= // ::= // diff --git a/clang/lib/AST/ODRHash.cpp b/clang/lib/AST/ODRHash.cpp index 2dbc259138a8..e159a1b00be5 100644 --- a/clang/lib/AST/ODRHash.cpp +++ b/clang/lib/AST/ODRHash.cpp @@ -944,6 +944,10 @@ public: VisitArrayType(T); } + void VisitArrayParameterType(const ArrayParameterType *T) { + VisitConstantArrayType(T); + } + void VisitDependentSizedArrayType(const DependentSizedArrayType *T) { AddStmt(T->getSizeExpr()); VisitArrayType(T); diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp index 8f3e26d46019..779d8a810820 100644 --- a/clang/lib/AST/Type.cpp +++ b/clang/lib/AST/Type.cpp @@ -1197,6 +1197,14 @@ public: return Ctx.getDecayedType(originalType); } + QualType VisitArrayParameterType(const ArrayParameterType *T) { + QualType ArrTy = VisitConstantArrayType(T); + if (ArrTy.isNull()) + return {}; + + return Ctx.getArrayParameterType(ArrTy); + } + SUGARED_TYPE_CLASS(TypeOfExpr) SUGARED_TYPE_CLASS(TypeOf) SUGARED_TYPE_CLASS(Decltype) @@ -4454,6 +4462,7 @@ static CachedProperties computeCachedProperties(const Type *T) { case Type::ConstantArray: case Type::IncompleteArray: case Type::VariableArray: + case Type::ArrayParameter: return Cache::get(cast(T)->getElementType()); case Type::Vector: case Type::ExtVector: @@ -4542,6 +4551,7 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) { case Type::ConstantArray: case Type::IncompleteArray: case Type::VariableArray: + case Type::ArrayParameter: return computeTypeLinkageInfo(cast(T)->getElementType()); case Type::Vector: case Type::ExtVector: @@ -4736,6 +4746,7 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const { case Type::Pipe: case Type::BitInt: case Type::DependentBitInt: + case Type::ArrayParameter: return false; } llvm_unreachable("bad type kind!"); diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp index 0aa1d9327d77..9d551ff83151 100644 --- a/clang/lib/AST/TypePrinter.cpp +++ b/clang/lib/AST/TypePrinter.cpp @@ -268,6 +268,7 @@ bool TypePrinter::canPrefixQualifiers(const Type *T, case Type::Adjusted: case Type::Decayed: + case Type::ArrayParameter: case Type::Pointer: case Type::BlockPointer: case Type::LValueReference: @@ -595,6 +596,16 @@ void TypePrinter::printDecayedBefore(const DecayedType *T, raw_ostream &OS) { printAdjustedBefore(T, OS); } +void TypePrinter::printArrayParameterAfter(const ArrayParameterType *T, + raw_ostream &OS) { + printConstantArrayAfter(T, OS); +} + +void TypePrinter::printArrayParameterBefore(const ArrayParameterType *T, + raw_ostream &OS) { + printConstantArrayBefore(T, OS); +} + void TypePrinter::printDecayedAfter(const DecayedType *T, raw_ostream &OS) { printAdjustedAfter(T, OS); } diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index a5fe39633679..9308528ac938 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -4719,7 +4719,8 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, } if (HasAggregateEvalKind && isa(E) && - cast(E)->getCastKind() == CK_LValueToRValue) { + cast(E)->getCastKind() == CK_LValueToRValue && + !type->isArrayParameterType()) { LValue L = EmitLValue(cast(E)->getSubExpr()); assert(L.isSimple()); args.addUncopiedAggregate(L, type); diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index 691fde8b0d8b..8c284c332171 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -3641,6 +3641,7 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) { case Type::ConstantArray: case Type::VariableArray: case Type::IncompleteArray: + case Type::ArrayParameter: return CreateType(cast(Ty), Unit); case Type::LValueReference: diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index 36872c0fedb7..e0d5575d57d0 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -5190,6 +5190,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { case CK_IntegralToFixedPoint: case CK_MatrixCast: case CK_HLSLVectorTruncation: + case CK_HLSLArrayRValue: return EmitUnsupportedLValue(E, "unexpected cast lvalue"); case CK_Dependent: diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index 143855aa84ca..1b9287ea2393 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -883,6 +883,9 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { [[fallthrough]]; + case CK_HLSLArrayRValue: + Visit(E->getSubExpr()); + break; case CK_NoOp: case CK_UserDefinedConversion: @@ -1524,6 +1527,7 @@ static bool castPreservesZero(const CastExpr *CE) { case CK_LValueToRValue: case CK_LValueToRValueBitCast: case CK_UncheckedDerivedToBase: + case CK_HLSLArrayRValue: return false; } llvm_unreachable("Unhandled clang::CastKind enum"); diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp index c3774d0cb75e..a793b214645c 100644 --- a/clang/lib/CodeGen/CGExprComplex.cpp +++ b/clang/lib/CodeGen/CGExprComplex.cpp @@ -616,6 +616,7 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op, case CK_IntegralToFixedPoint: case CK_MatrixCast: case CK_HLSLVectorTruncation: + case CK_HLSLArrayRValue: llvm_unreachable("invalid cast kind for complex value"); case CK_FloatingRealToComplex: diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp index 36d7493d9a6b..9f1b06eebf9e 100644 --- a/clang/lib/CodeGen/CGExprConstant.cpp +++ b/clang/lib/CodeGen/CGExprConstant.cpp @@ -1226,6 +1226,7 @@ public: case CK_ZeroToOCLOpaqueType: case CK_MatrixCast: case CK_HLSLVectorTruncation: + case CK_HLSLArrayRValue: return nullptr; } llvm_unreachable("Invalid CastKind"); diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp index 83247aa48f86..397b4977acc3 100644 --- a/clang/lib/CodeGen/CGExprScalar.cpp +++ b/clang/lib/CodeGen/CGExprScalar.cpp @@ -2329,6 +2329,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { case CK_FloatingComplexToIntegralComplex: case CK_ConstructorConversion: case CK_ToUnion: + case CK_HLSLArrayRValue: llvm_unreachable("scalar cast to non-scalar value"); case CK_LValueToRValue: diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index 44103884940f..90324de7268e 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -276,6 +276,7 @@ TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) { case Type::Record: case Type::ObjCObject: case Type::ObjCInterface: + case Type::ArrayParameter: return TEK_Aggregate; // We operate on atomic values according to their underlying type. @@ -2361,6 +2362,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { type = cast(ty)->getPointeeType(); break; + case Type::ArrayParameter: case Type::ConstantArray: case Type::IncompleteArray: // Losing element qualification here is fine. diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp index afadc29ab1b0..1568b6e6275b 100644 --- a/clang/lib/CodeGen/CodeGenTypes.cpp +++ b/clang/lib/CodeGen/CodeGenTypes.cpp @@ -590,6 +590,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) { ResultType = llvm::ArrayType::get(ResultType, 0); break; } + case Type::ArrayParameter: case Type::ConstantArray: { const ConstantArrayType *A = cast(Ty); llvm::Type *EltTy = ConvertTypeForMem(A->getElementType()); diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp index fd71317572f0..18acf7784f71 100644 --- a/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -3584,6 +3584,9 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty) { case Type::Pipe: llvm_unreachable("Pipe types shouldn't get here"); + case Type::ArrayParameter: + llvm_unreachable("Array Parameter types should not get here."); + case Type::Builtin: case Type::BitInt: // GCC treats vector and complex types as fundamental types. @@ -3868,6 +3871,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo( case Type::ConstantArray: case Type::IncompleteArray: case Type::VariableArray: + case Type::ArrayParameter: // Itanium C++ ABI 2.9.5p5: // abi::__array_type_info adds no data members to std::type_info. break; diff --git a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp index 22f2c47e1d6a..81797c8c4dc7 100644 --- a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp +++ b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp @@ -1000,6 +1000,7 @@ static bool rewriteToNumericBoxedExpression(const ObjCMessageExpr *Msg, case CK_LValueToRValue: case CK_NoOp: case CK_UserDefinedConversion: + case CK_HLSLArrayRValue: break; case CK_IntegralCast: { diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp index 72393bea6205..c9dbac0dfc33 100644 --- a/clang/lib/Sema/Sema.cpp +++ b/clang/lib/Sema/Sema.cpp @@ -653,6 +653,7 @@ ExprResult Sema::ImpCastExprToType(Expr *E, QualType Ty, case CK_FunctionToPointerDecay: case CK_ToVoid: case CK_NonAtomicToAtomic: + case CK_HLSLArrayRValue: break; } } diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index 091fc3e4836b..80b4257d9d83 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -658,8 +658,9 @@ ExprResult Sema::DefaultLvalueConversion(Expr *E) { QualType T = E->getType(); assert(!T.isNull() && "r-value conversion on typeless expression?"); - // lvalue-to-rvalue conversion cannot be applied to function or array types. - if (T->isFunctionType() || T->isArrayType()) + // lvalue-to-rvalue conversion cannot be applied to types that decay to + // pointers (i.e. function or array types). + if (T->canDecayToPointerType()) return E; // We don't want to throw lvalue-to-rvalue casts on top of @@ -4686,6 +4687,9 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T, case Type::Decayed: T = cast(Ty)->getPointeeType(); break; + case Type::ArrayParameter: + T = cast(Ty)->getElementType(); + break; case Type::Pointer: T = cast(Ty)->getPointeeType(); break; @@ -12908,6 +12912,8 @@ static ImplicitConversionKind castKindToImplicitConversionKind(CastKind CK) { case CK_IntegralComplexToReal: case CK_IntegralRealToComplex: return ICK_Complex_Real; + case CK_HLSLArrayRValue: + return ICK_HLSL_Array_RValue; } } diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp index 51c8e04bee8c..76bb78aa8b54 100644 --- a/clang/lib/Sema/SemaExprCXX.cpp +++ b/clang/lib/Sema/SemaExprCXX.cpp @@ -4416,6 +4416,13 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType, .get(); break; + case ICK_HLSL_Array_RValue: + FromType = Context.getArrayParameterType(FromType); + From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue, + /*BasePath=*/nullptr, CCK) + .get(); + break; + case ICK_Function_To_Pointer: FromType = Context.getPointerType(FromType); From = ImpCastExprToType(From, FromType, CK_FunctionToPointerDecay, @@ -4793,6 +4800,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType, case ICK_Num_Conversion_Kinds: case ICK_C_Only_Conversion: case ICK_Incompatible_Pointer_Conversion: + case ICK_HLSL_Array_RValue: llvm_unreachable("Improper second standard conversion"); } diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index dce225a7204d..777f89c70f87 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -6269,7 +6269,10 @@ void InitializationSequence::InitializeFrom(Sema &S, // initializer is a string literal, see 8.5.2. // - Otherwise, if the destination type is an array, the program is // ill-formed. - if (const ArrayType *DestAT = Context.getAsArrayType(DestType)) { + // - Except in HLSL, where non-decaying array parameters behave like + // non-array types for initialization. + if (DestType->isArrayType() && !DestType->isArrayParameterType()) { + const ArrayType *DestAT = Context.getAsArrayType(DestType); if (Initializer && isa(DestAT)) { SetFailed(FK_VariableLengthArrayHasInitializer); return; diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp index d3a9c7abd0e9..38237ee57807 100644 --- a/clang/lib/Sema/SemaLookup.cpp +++ b/clang/lib/Sema/SemaLookup.cpp @@ -3243,6 +3243,10 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) { case Type::Pipe: T = cast(T)->getElementType().getTypePtr(); continue; + + // Array parameter types are treated as fundamental types. + case Type::ArrayParameter: + break; } if (Queue.empty()) diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp index 51450e486eae..16d54c1ffe5f 100644 --- a/clang/lib/Sema/SemaOverload.cpp +++ b/clang/lib/Sema/SemaOverload.cpp @@ -160,6 +160,7 @@ ImplicitConversionRank clang::GetConversionRank(ImplicitConversionKind Kind) { ICR_C_Conversion_Extension, ICR_Conversion, ICR_Conversion, + ICR_Conversion, }; static_assert(std::size(Rank) == (int)ICK_Num_Conversion_Kinds); return Rank[(int)Kind]; @@ -201,6 +202,7 @@ static const char *GetImplicitConversionName(ImplicitConversionKind Kind) { "Incompatible pointer conversion", "Fixed point conversion", "HLSL vector truncation", + "Non-decaying array conversion", }; static_assert(std::size(Name) == (int)ICK_Num_Conversion_Kinds); return Name[Kind]; @@ -2131,8 +2133,7 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType, // A glvalue (3.10) of a non-function, non-array type T can // be converted to a prvalue. bool argIsLValue = From->isGLValue(); - if (argIsLValue && - !FromType->isFunctionType() && !FromType->isArrayType() && + if (argIsLValue && !FromType->canDecayToPointerType() && S.Context.getCanonicalType(FromType) != S.Context.OverloadTy) { SCS.First = ICK_Lvalue_To_Rvalue; @@ -2147,6 +2148,19 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType, // is T (C++ 4.1p1). C++ can't get here with class types; in C, we // just strip the qualifiers because they don't matter. FromType = FromType.getUnqualifiedType(); + } else if (S.getLangOpts().HLSL && FromType->isConstantArrayType() && + ToType->isArrayParameterType()) { + // HLSL constant array parameters do not decay, so if the argument is a + // constant array and the parameter is an ArrayParameterType we have special + // handling here. + FromType = S.Context.getArrayParameterType(FromType); + if (S.Context.getCanonicalType(FromType) != + S.Context.getCanonicalType(ToType)) + return false; + + SCS.First = ICK_HLSL_Array_RValue; + SCS.setAllToTypes(ToType); + return true; } else if (FromType->isArrayType()) { // Array-to-pointer conversion (C++ 4.2) SCS.First = ICK_Array_To_Pointer; @@ -6100,6 +6114,7 @@ static bool CheckConvertedConstantConversions(Sema &S, case ICK_Lvalue_To_Rvalue: case ICK_Array_To_Pointer: case ICK_Function_To_Pointer: + case ICK_HLSL_Array_RValue: llvm_unreachable("found a first conversion kind in Second"); case ICK_Function_Conversion: diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index 9cd19d711af4..1a2d5e9310db 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -6898,6 +6898,11 @@ bool UnnamedLocalNoLinkageFinder::VisitBitIntType(const BitIntType *T) { return false; } +bool UnnamedLocalNoLinkageFinder::VisitArrayParameterType( + const ArrayParameterType *T) { + return VisitConstantArrayType(T); +} + bool UnnamedLocalNoLinkageFinder::VisitDependentBitIntType( const DependentBitIntType *T) { return false; diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp index 9a55881f6442..716660244537 100644 --- a/clang/lib/Sema/SemaTemplateDeduction.cpp +++ b/clang/lib/Sema/SemaTemplateDeduction.cpp @@ -2277,6 +2277,7 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch( case Type::DependentTemplateSpecialization: case Type::PackExpansion: case Type::Pipe: + case Type::ArrayParameter: // No template argument deduction for these types return TemplateDeductionResult::Success; @@ -6355,11 +6356,11 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T, case Type::ConstantArray: case Type::IncompleteArray: + case Type::ArrayParameter: MarkUsedTemplateParameters(Ctx, cast(T)->getElementType(), OnlyDeduced, Depth, Used); break; - case Type::Vector: case Type::ExtVector: MarkUsedTemplateParameters(Ctx, diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp index fd94caa4e1d4..d88895d35294 100644 --- a/clang/lib/Sema/SemaType.cpp +++ b/clang/lib/Sema/SemaType.cpp @@ -6505,6 +6505,9 @@ namespace { void VisitDecayedTypeLoc(DecayedTypeLoc TL) { llvm_unreachable("decayed type locs not expected here!"); } + void VisitArrayParameterTypeLoc(ArrayParameterTypeLoc TL) { + llvm_unreachable("array parameter type locs not expected here!"); + } void VisitAttributedTypeLoc(AttributedTypeLoc TL) { fillAttributedTypeLoc(TL, State); diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h index 80a10647ca5d..eace1bfdff5a 100644 --- a/clang/lib/Sema/TreeTransform.h +++ b/clang/lib/Sema/TreeTransform.h @@ -5243,6 +5243,23 @@ QualType TreeTransform::TransformDecayedType(TypeLocBuilder &TLB, return Result; } +template +QualType +TreeTransform::TransformArrayParameterType(TypeLocBuilder &TLB, + ArrayParameterTypeLoc TL) { + QualType OriginalType = getDerived().TransformType(TLB, TL.getElementLoc()); + if (OriginalType.isNull()) + return QualType(); + + QualType Result = TL.getType(); + if (getDerived().AlwaysRebuild() || + OriginalType != TL.getElementLoc().getType()) + Result = SemaRef.Context.getArrayParameterType(OriginalType); + TLB.push(Result); + // Nothing to set for ArrayParameterTypeLoc. + return Result; +} + template QualType TreeTransform::TransformPointerType(TypeLocBuilder &TLB, PointerTypeLoc TL) { diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 28e8d27fef08..004859ed22bf 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -6810,6 +6810,10 @@ void TypeLocReader::VisitAdjustedTypeLoc(AdjustedTypeLoc TL) { // nothing to do } +void TypeLocReader::VisitArrayParameterTypeLoc(ArrayParameterTypeLoc TL) { + // nothing to do +} + void TypeLocReader::VisitMacroQualifiedTypeLoc(MacroQualifiedTypeLoc TL) { TL.setExpansionLoc(readSourceLocation()); } diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 2438fbc16606..a2668e61c51d 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -318,6 +318,10 @@ void TypeLocWriter::VisitAdjustedTypeLoc(AdjustedTypeLoc TL) { // nothing to do } +void TypeLocWriter::VisitArrayParameterTypeLoc(ArrayParameterTypeLoc TL) { + // nothing to do +} + void TypeLocWriter::VisitBlockPointerTypeLoc(BlockPointerTypeLoc TL) { addSourceLocation(TL.getCaretLoc()); } diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp index c3fc56ac30ee..7a900780384a 100644 --- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp +++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp @@ -330,7 +330,8 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex, case CK_ConstructorConversion: case CK_UserDefinedConversion: case CK_FunctionToPointerDecay: - case CK_BuiltinFnToFnPtr: { + case CK_BuiltinFnToFnPtr: + case CK_HLSLArrayRValue: { // Copy the SVal of Ex to CastE. ProgramStateRef state = Pred->getState(); const LocationContext *LCtx = Pred->getLocationContext(); diff --git a/clang/test/CodeGenHLSL/ArrayTemporary.hlsl b/clang/test/CodeGenHLSL/ArrayTemporary.hlsl new file mode 100644 index 000000000000..63a30b61440e --- /dev/null +++ b/clang/test/CodeGenHLSL/ArrayTemporary.hlsl @@ -0,0 +1,104 @@ +// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s + +void fn(float x[2]) { } + +// CHECK-LABEL: define void {{.*}}call{{.*}} +// CHECK: [[Arr:%.*]] = alloca [2 x float] +// CHECK: [[Tmp:%.*]] = alloca [2 x float] +// CHECK: call void @llvm.memset.p0.i32(ptr align 4 [[Arr]], i8 0, i32 8, i1 false) +// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[Arr]], i32 8, i1 false) +// CHECK: call void {{.*}}fn{{.*}}(ptr noundef byval([2 x float]) align 4 [[Tmp]]) +void call() { + float Arr[2] = {0, 0}; + fn(Arr); +} + +struct Obj { + float V; + int X; +}; + +void fn2(Obj O[4]) { } + +// CHECK-LABEL: define void {{.*}}call2{{.*}} +// CHECK: [[Arr:%.*]] = alloca [4 x %struct.Obj] +// CHECK: [[Tmp:%.*]] = alloca [4 x %struct.Obj] +// CHECK: call void @llvm.memset.p0.i32(ptr align 4 [[Arr]], i8 0, i32 32, i1 false) +// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[Arr]], i32 32, i1 false) +// CHECK: call void {{.*}}fn2{{.*}}(ptr noundef byval([4 x %struct.Obj]) align 4 [[Tmp]]) +void call2() { + Obj Arr[4] = {}; + fn2(Arr); +} + + +void fn3(float x[2][2]) { } + +// CHECK-LABEL: define void {{.*}}call3{{.*}} +// CHECK: [[Arr:%.*]] = alloca [2 x [2 x float]] +// CHECK: [[Tmp:%.*]] = alloca [2 x [2 x float]] +// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr]], ptr align 4 {{.*}}, i32 16, i1 false) +// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[Arr]], i32 16, i1 false) +// CHECK: call void {{.*}}fn3{{.*}}(ptr noundef byval([2 x [2 x float]]) align 4 [[Tmp]]) +void call3() { + float Arr[2][2] = {{0, 0}, {1,1}}; + fn3(Arr); +} + +// CHECK-LABEL: define void {{.*}}call4{{.*}}(ptr +// CHECK-SAME: noundef byval([2 x [2 x float]]) align 4 [[Arr:%.*]]) +// CHECK: [[Tmp:%.*]] = alloca [2 x [2 x float]] +// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[Arr]], i32 16, i1 false) +// CHECK: call void {{.*}}fn3{{.*}}(ptr noundef byval([2 x [2 x float]]) align 4 [[Tmp]]) + +void call4(float Arr[2][2]) { + fn3(Arr); +} + +// Verify that each template instantiation codegens to a unique and correctly +// mangled function name. + +// CHECK-LABEL: define void {{.*}}template_call{{.*}}(ptr + +// CHECK-SAME: noundef byval([2 x float]) align 4 [[FA2:%[0-9A-Z]+]], +// CHECK-SAME: ptr noundef byval([4 x float]) align 4 [[FA4:%[0-9A-Z]+]], +// CHECK-SAME: ptr noundef byval([3 x i32]) align 4 [[IA3:%[0-9A-Z]+]] + +// CHECK: [[Tmp1:%.*]] = alloca [2 x float] +// CHECK: [[Tmp2:%.*]] = alloca [4 x float] +// CHECK: [[Tmp3:%.*]] = alloca [3 x i32] +// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp1]], ptr align 4 [[FA2]], i32 8, i1 false) +// CHECK: call void @"??$template_fn@$$BY01M@@YAXY01M@Z"(ptr noundef byval([2 x float]) align 4 [[Tmp1]]) +// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp2]], ptr align 4 [[FA4]], i32 16, i1 false) +// CHECK: call void @"??$template_fn@$$BY03M@@YAXY03M@Z"(ptr noundef byval([4 x float]) align 4 [[Tmp2]]) +// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp3]], ptr align 4 [[IA3]], i32 12, i1 false) +// CHECK: call void @"??$template_fn@$$BY02H@@YAXY02H@Z"(ptr noundef byval([3 x i32]) align 4 [[Tmp3]]) + +template +void template_fn(T Val) {} + +void template_call(float FA2[2], float FA4[4], int IA3[3]) { + template_fn(FA2); + template_fn(FA4); + template_fn(IA3); +} + + +// Verify that Array parameter element access correctly codegens. +// CHECK-LABEL: define void {{.*}}element_access{{.*}}(ptr +// CHECK-SAME: noundef byval([2 x float]) align 4 [[FA2:%[0-9A-Z]+]] + +// CHECK: [[Addr:%.*]] = getelementptr inbounds [2 x float], ptr [[FA2]], i32 0, i32 0 +// CHECK: [[Tmp:%.*]] = load float, ptr [[Addr]] +// CHECK: call void @"??$template_fn@M@@YAXM@Z"(float noundef [[Tmp]]) + +// CHECK: [[Idx0:%.*]] = getelementptr inbounds [2 x float], ptr [[FA2]], i32 0, i32 0 +// CHECK: [[Val0:%.*]] = load float, ptr [[Idx0]] +// CHECK: [[Sum:%.*]] = fadd float [[Val0]], 5.000000e+00 +// CHECK: [[Idx1:%.*]] = getelementptr inbounds [2 x float], ptr [[FA2]], i32 0, i32 1 +// CHECK: store float [[Sum]], ptr [[Idx1]] + +void element_access(float FA2[2]) { + template_fn(FA2[0]); + FA2[1] = FA2[0] + 5; +} diff --git a/clang/test/SemaHLSL/ArrayParams.hlsl b/clang/test/SemaHLSL/ArrayParams.hlsl new file mode 100644 index 000000000000..75af9d775206 --- /dev/null +++ b/clang/test/SemaHLSL/ArrayParams.hlsl @@ -0,0 +1,29 @@ +// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -verify + +void fn(int I[5]); // #fn +void fn2(int I[3][3]); // #fn2 + +void call() { + float F[5]; + double D[4]; + int Long[9]; + int Short[4]; + int Same[5]; + + fn(F); // expected-error{{no matching function for call to 'fn'}} + // expected-note@#fn{{candidate function not viable: no known conversion from 'float[5]' to 'int[5]' for 1st argument}} + + fn(D); // expected-error{{no matching function for call to 'fn'}} + // expected-note@#fn{{candidate function not viable: no known conversion from 'double[4]' to 'int[5]' for 1st argument}} + + fn(Long); // expected-error{{no matching function for call to 'fn'}} + // expected-note@#fn{{candidate function not viable: no known conversion from 'int[9]' to 'int[5]' for 1st argument}} + + fn(Short); // expected-error{{no matching function for call to 'fn'}} + // expected-note@#fn{{candidate function not viable: no known conversion from 'int[4]' to 'int[5]' for 1st argument}} + + fn(Same); // totally fine, nothing to see here. + + fn2(Long); // expected-error{{no matching function for call to 'fn2'}} + // expected-note@#fn2{{candidate function not viable: no known conversion from 'int[9]' to 'int[3][3]' for 1st argument}} +} diff --git a/clang/test/SemaHLSL/ArrayTemporary.hlsl b/clang/test/SemaHLSL/ArrayTemporary.hlsl new file mode 100644 index 000000000000..dff9aff7d9b2 --- /dev/null +++ b/clang/test/SemaHLSL/ArrayTemporary.hlsl @@ -0,0 +1,95 @@ +// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -ast-dump %s | FileCheck %s + +void fn(float x[2]) { } + +// CHECK: CallExpr {{.*}} 'void' +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float[2])' +// CHECK-NEXT: DeclRefExpr {{.*}} 'void (float[2])' lvalue Function {{.*}} 'fn' 'void (float[2])' +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float[2]' + +void call() { + float Arr[2] = {0, 0}; + fn(Arr); +} + +struct Obj { + float V; + int X; +}; + +void fn2(Obj O[4]) { } + +// CHECK: CallExpr {{.*}} 'void' +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(Obj[4])' +// CHECK-NEXT: DeclRefExpr {{.*}} 'void (Obj[4])' lvalue Function {{.*}} 'fn2' 'void (Obj[4])' +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'Obj[4]' + +void call2() { + Obj Arr[4] = {}; + fn2(Arr); +} + + +void fn3(float x[2][2]) { } + +// CHECK: CallExpr {{.*}} 'void' +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float[2][2])' +// CHECK-NEXT: DeclRefExpr {{.*}} 'void (float[2][2])' lvalue Function {{.*}} 'fn3' 'void (float[2][2])' +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float[2][2]' + +void call3() { + float Arr[2][2] = {{0, 0}, {1,1}}; + fn3(Arr); +} + +// This template function should be instantiated 3 times for the different array +// types and lengths. + +// CHECK: FunctionTemplateDecl {{.*}} template_fn +// CHECK-NEXT: TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T +// CHECK-NEXT: FunctionDecl {{.*}} template_fn 'void (T)' +// CHECK-NEXT: ParmVarDecl {{.*}} Val 'T' + +// CHECK: FunctionDecl {{.*}} used template_fn 'void (float[2])' implicit_instantiation +// CHECK-NEXT: TemplateArgument type 'float[2]' +// CHECK-NEXT: ArrayParameterType {{.*}} 'float[2]' 2 +// CHECK-NEXT: BuiltinType {{.*}} 'float' +// CHECK-NEXT: ParmVarDecl {{.*}} Val 'float[2]' + +// CHECK: FunctionDecl {{.*}} used template_fn 'void (float[4])' implicit_instantiation +// CHECK-NEXT: TemplateArgument type 'float[4]' +// CHECK-NEXT: ArrayParameterType {{.*}} 'float[4]' 4 +// CHECK-NEXT: BuiltinType {{.*}} 'float' +// CHECK-NEXT: ParmVarDecl {{.*}} Val 'float[4]' + +// CHECK: FunctionDecl {{.*}} used template_fn 'void (int[3])' implicit_instantiation +// CHECK-NEXT: TemplateArgument type 'int[3]' +// CHECK-NEXT: ArrayParameterType {{.*}} 'int[3]' 3 +// CHECK-NEXT: BuiltinType {{.*}} 'int' +// CHECK-NEXT: ParmVarDecl {{.*}} Val 'int[3]' + +template +void template_fn(T Val) {} + +// CHECK: FunctionDecl {{.*}} call 'void (float[2], float[4], int[3])' +// CHECK: CallExpr {{.*}} 'void' +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float[2])' +// CHECK-NEXT: DeclRefExpr {{.*}} 'void (float[2])' lvalue Function {{.*}} 'template_fn' 'void (float[2])' (FunctionTemplate {{.*}} 'template_fn') +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float[2]' +// CHECK-NEXT: DeclRefExpr {{.*}} 'float[2]' lvalue ParmVar {{.*}} 'FA2' 'float[2]' +// CHECK-NEXT: CallExpr {{.*}} 'void' +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float[4])' +// CHECK-NEXT: DeclRefExpr {{.*}} 'void (float[4])' lvalue Function {{.*}} 'template_fn' 'void (float[4])' (FunctionTemplate {{.*}} 'template_fn') +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float[4]' +// CHECK-NEXT: DeclRefExpr {{.*}} 'float[4]' lvalue ParmVar {{.*}} 'FA4' 'float[4]' +// CHECK-NEXT: CallExpr {{.*}} 'void' +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(int[3])' +// CHECK-NEXT: DeclRefExpr {{.*}} 'void (int[3])' lvalue Function {{.*}} 'template_fn' 'void (int[3])' (FunctionTemplate {{.*}} 'template_fn') +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int[3]' +// CHECK-NEXT: DeclRefExpr {{.*}} 'int[3]' lvalue ParmVar {{.*}} 'IA3' 'int[3]' + +void call(float FA2[2], float FA4[4], int IA3[3]) { + template_fn(FA2); + template_fn(FA4); + template_fn(IA3); +} diff --git a/clang/test/SemaHLSL/ArrayTemporary.ll b/clang/test/SemaHLSL/ArrayTemporary.ll new file mode 100644 index 000000000000..5eec0094865b --- /dev/null +++ b/clang/test/SemaHLSL/ArrayTemporary.ll @@ -0,0 +1,76 @@ +; ModuleID = '/Users/cbieneman/dev/llvm-project/clang/test/SemaHLSL/ArrayTemporary.hlsl' +source_filename = "/Users/cbieneman/dev/llvm-project/clang/test/SemaHLSL/ArrayTemporary.hlsl" +target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-pc-shadermodel6.3-library" + +%struct.Obj = type { float, i32 } + +@"__const.?call3@@YAXXZ.Arr" = private unnamed_addr constant [2 x [2 x float]] [[2 x float] zeroinitializer, [2 x float] [float 1.000000e+00, float 1.000000e+00]], align 4 + +; Function Attrs: noinline nounwind optnone +define void @"?fn@@YAXY01M@Z"(ptr noundef byval([2 x float]) align 4 %x) #0 { +entry: + ret void +} + +; Function Attrs: noinline nounwind optnone +define void @"?call@@YAXXZ"() #0 { +entry: + %Arr = alloca [2 x float], align 4 + %agg.tmp = alloca [2 x float], align 4 + call void @llvm.memset.p0.i32(ptr align 4 %Arr, i8 0, i32 8, i1 false) + call void @llvm.memcpy.p0.p0.i32(ptr align 4 %agg.tmp, ptr align 4 %Arr, i32 8, i1 false) + call void @"?fn@@YAXY01M@Z"(ptr noundef byval([2 x float]) align 4 %agg.tmp) + ret void +} + +; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: write) +declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1 immarg) #1 + +; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite) +declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) #2 + +; Function Attrs: noinline nounwind optnone +define void @"?fn2@@YAXY03UObj@@@Z"(ptr noundef byval([4 x %struct.Obj]) align 4 %O) #0 { +entry: + ret void +} + +; Function Attrs: noinline nounwind optnone +define void @"?call2@@YAXXZ"() #0 { +entry: + %Arr = alloca [4 x %struct.Obj], align 4 + %agg.tmp = alloca [4 x %struct.Obj], align 4 + call void @llvm.memset.p0.i32(ptr align 4 %Arr, i8 0, i32 32, i1 false) + call void @llvm.memcpy.p0.p0.i32(ptr align 4 %agg.tmp, ptr align 4 %Arr, i32 32, i1 false) + call void @"?fn2@@YAXY03UObj@@@Z"(ptr noundef byval([4 x %struct.Obj]) align 4 %agg.tmp) + ret void +} + +; Function Attrs: noinline nounwind optnone +define void @"?fn3@@YAXY111M@Z"(ptr noundef byval([2 x [2 x float]]) align 4 %x) #0 { +entry: + ret void +} + +; Function Attrs: noinline nounwind optnone +define void @"?call3@@YAXXZ"() #0 { +entry: + %Arr = alloca [2 x [2 x float]], align 4 + %agg.tmp = alloca [2 x [2 x float]], align 4 + call void @llvm.memcpy.p0.p0.i32(ptr align 4 %Arr, ptr align 4 @"__const.?call3@@YAXXZ.Arr", i32 16, i1 false) + call void @llvm.memcpy.p0.p0.i32(ptr align 4 %agg.tmp, ptr align 4 %Arr, i32 16, i1 false) + call void @"?fn3@@YAXY111M@Z"(ptr noundef byval([2 x [2 x float]]) align 4 %agg.tmp) + ret void +} + +attributes #0 = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nounwind willreturn memory(argmem: write) } +attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) } + +!llvm.module.flags = !{!0, !1} +!llvm.ident = !{!2} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{i32 4, !"dx.disable_optimizations", i32 1} +!2 = !{!"clang version 19.0.0git (git@github.com:llvm/llvm-project.git 64e1c15c520cf11114ef2ddd887e76560903db2b)"} diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp index 454ee1e42aed..00ddc4f24500 100644 --- a/clang/tools/libclang/CIndex.cpp +++ b/clang/tools/libclang/CIndex.cpp @@ -1923,6 +1923,7 @@ DEFAULT_TYPELOC_IMPL(ConstantArray, ArrayType) DEFAULT_TYPELOC_IMPL(IncompleteArray, ArrayType) DEFAULT_TYPELOC_IMPL(VariableArray, ArrayType) DEFAULT_TYPELOC_IMPL(DependentSizedArray, ArrayType) +DEFAULT_TYPELOC_IMPL(ArrayParameter, ConstantArrayType) DEFAULT_TYPELOC_IMPL(DependentAddressSpace, Type) DEFAULT_TYPELOC_IMPL(DependentVector, Type) DEFAULT_TYPELOC_IMPL(DependentSizedExtVector, Type) -- GitLab From 53d256bb2790a0e5300f275345c864930b7e6f82 Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 1 Apr 2024 13:25:26 -0400 Subject: [PATCH 0122/1690] Update the "Current Status" section of the website to be current. (#84507) The section discusses the reasons for the libraries inception more than a decade ago. Now it discusses the progess libc++ has made, and the many impressive acomplishments our contributors have brought it. The initial section remains below. --------- Co-authored-by: Louis Dionne --- libcxx/docs/index.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/libcxx/docs/index.rst b/libcxx/docs/index.rst index 2a7e47dfe6d8..db55c6f02a3d 100644 --- a/libcxx/docs/index.rst +++ b/libcxx/docs/index.rst @@ -66,6 +66,25 @@ Getting Started with libc++ Current Status ============== +libc++ has become the default C++ Standard Library implementation for many major platforms, including Apple's macOS, +iOS, watchOS, and tvOS, Google Search, the Android operating system, and FreeBSD. As a result, libc++ has an estimated +user base of over 1 billion daily active users. + +Since its inception, libc++ has focused on delivering high performance, standards-conformance, and portability. It has +been extensively tested and optimized, making it robust and production ready. libc++ fully implements C++11 and C++14, +with C++17, C++20, C++23, and C++26 features being actively developed and making steady progress. + +libc++ is continuously integrated and tested on a wide range of platforms and configurations, ensuring its reliability +and compatibility across various systems. The library's extensive test suite and rigorous quality assurance process have +made it a top choice for platform providers looking to offer their users a robust and efficient C++ Standard Library. + +As an open-source project, libc++ benefits from a vibrant community of contributors who work together to improve the +library and add new features. This ongoing development and support ensure that libc++ remains at the forefront of +C++ standardization efforts and continues to meet the evolving needs of C++ developers worldwide. + + +History +------- After its initial introduction, many people have asked "why start a new library instead of contributing to an existing library?" (like Apache's libstdcxx, GNU's libstdc++, STLport, etc). There are many contributing -- GitLab From 2cfd7d433be0831c6e2a248a4b828f7aedcaeaa0 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Mon, 1 Apr 2024 10:26:33 -0700 Subject: [PATCH 0123/1690] [libc] fixup missing include for fullbuild (#87266) Fixes #86928 --- libc/src/stdio/fseeko.h | 1 + libc/src/stdio/ftello.h | 1 + 2 files changed, 2 insertions(+) diff --git a/libc/src/stdio/fseeko.h b/libc/src/stdio/fseeko.h index 77fb41215c31..3202ed2f97d0 100644 --- a/libc/src/stdio/fseeko.h +++ b/libc/src/stdio/fseeko.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_STDIO_FSEEKO_H #include +#include namespace LIBC_NAMESPACE { diff --git a/libc/src/stdio/ftello.h b/libc/src/stdio/ftello.h index 5ab17f9244a5..0fdf13ab6bdb 100644 --- a/libc/src/stdio/ftello.h +++ b/libc/src/stdio/ftello.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_STDIO_FTELLO_H #include +#include namespace LIBC_NAMESPACE { -- GitLab From a54930e696a275ac3947484f44d770cd587ce147 Mon Sep 17 00:00:00 2001 From: Peiming Liu Date: Mon, 1 Apr 2024 10:30:36 -0700 Subject: [PATCH 0124/1690] [mlir][sparse] allow YieldOp to yield multiple values. (#87261) --- .../SparseTensor/IR/SparseTensorOps.td | 25 +++++++++++++++---- .../SparseTensor/IR/SparseTensorDialect.cpp | 18 +++---------- .../Transforms/SparseReinterpretMap.cpp | 5 ++-- .../lib/Dialect/SparseTensor/Utils/Merger.cpp | 4 +-- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td index 29cf8c32447e..5df8a176459b 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td +++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td @@ -1278,8 +1278,10 @@ def SparseTensor_SelectOp : SparseTensor_Op<"select", [Pure, SameOperandsAndResu let hasVerifier = 1; } -def SparseTensor_YieldOp : SparseTensor_Op<"yield", [Pure, Terminator]>, - Arguments<(ins Optional:$result)> { +def SparseTensor_YieldOp : SparseTensor_Op<"yield", [Pure, Terminator, + ParentOneOf<["BinaryOp", "UnaryOp", "ReduceOp", "SelectOp", + "ForeachOp"]>]>, + Arguments<(ins Variadic:$results)> { let summary = "Yield from sparse_tensor set-like operations"; let description = [{ Yields a value from within a `binary`, `unary`, `reduce`, @@ -1302,14 +1304,27 @@ def SparseTensor_YieldOp : SparseTensor_Op<"yield", [Pure, Terminator]>, let builders = [ OpBuilder<(ins), [{ - build($_builder, $_state, Value()); + build($_builder, $_state, ValueRange()); + }]>, + OpBuilder<(ins "Value":$yieldVal), + [{ + build($_builder, $_state, ValueRange(yieldVal)); }]> ]; + let extraClassDeclaration = [{ + Value getSingleResult() { + assert(hasSingleResult()); + return getResults().front(); + } + bool hasSingleResult() { + return getResults().size() == 1; + } + }]; + let assemblyFormat = [{ - $result attr-dict `:` type($result) + $results attr-dict `:` type($results) }]; - let hasVerifier = 1; } def SparseTensor_ForeachOp : SparseTensor_Op<"foreach", diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp index 6da51bb6b9ca..e4d93c5623b9 100644 --- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp +++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp @@ -1591,7 +1591,8 @@ static LogicalResult verifyNumBlockArgs(T *op, Region ®ion, if (!yield) return op->emitError() << regionName << " region must end with sparse_tensor.yield"; - if (!yield.getResult() || yield.getResult().getType() != outputType) + if (!yield.hasSingleResult() || + yield.getSingleResult().getType() != outputType) return op->emitError() << regionName << " region yield type mismatch"; return success(); @@ -1654,7 +1655,8 @@ LogicalResult UnaryOp::verify() { // Absent branch can only yield invariant values. Block *absentBlock = &absent.front(); Block *parent = getOperation()->getBlock(); - Value absentVal = cast(absentBlock->getTerminator()).getResult(); + Value absentVal = + cast(absentBlock->getTerminator()).getSingleResult(); if (auto arg = dyn_cast(absentVal)) { if (arg.getOwner() == parent) return emitError("absent region cannot yield linalg argument"); @@ -1907,18 +1909,6 @@ LogicalResult SortOp::verify() { return success(); } -LogicalResult YieldOp::verify() { - // Check for compatible parent. - auto *parentOp = (*this)->getParentOp(); - if (isa(parentOp) || isa(parentOp) || - isa(parentOp) || isa(parentOp) || - isa(parentOp)) - return success(); - - return emitOpError("expected parent op to be sparse_tensor unary, binary, " - "reduce, select or foreach"); -} - /// Materialize a single constant operation from a given attribute value with /// the desired resultant type. Operation *SparseTensorDialect::materializeConstant(OpBuilder &builder, diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp index 14ea07f0b54b..9c0fc60877d8 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp @@ -764,9 +764,10 @@ struct ForeachOpDemapper if (numInitArgs != 0) { rewriter.setInsertionPointToEnd(body); auto yield = llvm::cast(body->getTerminator()); - if (auto stt = tryGetSparseTensorType(yield.getResult()); + if (auto stt = tryGetSparseTensorType(yield.getSingleResult()); stt && !stt->isIdentity()) { - Value y = genDemap(rewriter, stt->getEncoding(), yield.getResult()); + Value y = + genDemap(rewriter, stt->getEncoding(), yield.getSingleResult()); rewriter.create(loc, y); rewriter.eraseOp(yield); } diff --git a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp index 72b722c69ae3..9c0aed3c18ef 100644 --- a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp +++ b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp @@ -1031,7 +1031,7 @@ LatSetId Merger::buildLattices(ExprId e, LoopId i) { // invariant on the right. Block &absentBlock = absentRegion.front(); YieldOp absentYield = cast(absentBlock.getTerminator()); - const Value absentVal = absentYield.getResult(); + const Value absentVal = absentYield.getSingleResult(); const ExprId rhs = addInvariantExp(absentVal); return disjSet(e, child0, buildLattices(rhs, i), unop); } @@ -1500,7 +1500,7 @@ static Value insertYieldOp(RewriterBase &rewriter, Location loc, Region ®ion, // Merge cloned block and return yield value. Operation *placeholder = rewriter.create(loc, 0); rewriter.inlineBlockBefore(&tmpRegion.front(), placeholder, vals); - Value val = clonedYield.getResult(); + Value val = clonedYield.getSingleResult(); rewriter.eraseOp(clonedYield); rewriter.eraseOp(placeholder); return val; -- GitLab From 2be722587f5987891ed8b2904a03f983e987f226 Mon Sep 17 00:00:00 2001 From: lntue <35648136+lntue@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:31:07 -0400 Subject: [PATCH 0125/1690] [libc][math] Implement atan2f correctly rounded to all rounding modes. (#86716) We compute atan2f(y, x) in 2 stages: - Fast step: perform computations in double precision , with relative errors < 2^-50 - Accurate step: if the result from the Fast step fails Ziv's rounding test, then we perform computations in double-double precision, with relative errors < 2^-100. On Ryzen 5900X, worst-case latency is ~ 200 clocks, compared to average latency ~ 60 clocks, and average reciprocal throughput ~ 20 clocks. --- libc/config/baremetal/arm/entrypoints.txt | 1 + libc/config/baremetal/riscv/entrypoints.txt | 1 + libc/config/darwin/arm/entrypoints.txt | 1 + libc/config/linux/aarch64/entrypoints.txt | 1 + libc/config/linux/arm/entrypoints.txt | 1 + libc/config/linux/riscv/entrypoints.txt | 1 + libc/config/linux/x86_64/entrypoints.txt | 1 + libc/config/windows/entrypoints.txt | 1 + libc/docs/math/index.rst | 3 +- libc/spec/stdc.td | 4 + libc/src/math/generic/CMakeLists.txt | 18 ++ libc/src/math/generic/atan2f.cpp | 306 ++++++++++++++++++++ libc/src/math/generic/atanf.cpp | 22 +- libc/src/math/generic/inv_trigf_utils.cpp | 117 ++++---- libc/src/math/generic/inv_trigf_utils.h | 7 +- libc/test/src/math/CMakeLists.txt | 13 + libc/test/src/math/atan2f_test.cpp | 133 +++++++++ libc/test/src/math/atanf_test.cpp | 5 +- libc/test/src/math/smoke/CMakeLists.txt | 12 + libc/test/src/math/smoke/atan2f_test.cpp | 50 ++++ libc/utils/MPFRWrapper/MPFRUtils.cpp | 8 + libc/utils/MPFRWrapper/MPFRUtils.h | 1 + 22 files changed, 634 insertions(+), 73 deletions(-) create mode 100644 libc/src/math/generic/atan2f.cpp create mode 100644 libc/test/src/math/atan2f_test.cpp create mode 100644 libc/test/src/math/smoke/atan2f_test.cpp diff --git a/libc/config/baremetal/arm/entrypoints.txt b/libc/config/baremetal/arm/entrypoints.txt index 17ce56e228a6..9e21f5c20d92 100644 --- a/libc/config/baremetal/arm/entrypoints.txt +++ b/libc/config/baremetal/arm/entrypoints.txt @@ -207,6 +207,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.acoshf libc.src.math.asinf libc.src.math.asinhf + libc.src.math.atan2f libc.src.math.atanf libc.src.math.atanhf libc.src.math.ceil diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt index 39756e1ee29f..7664937da0f6 100644 --- a/libc/config/baremetal/riscv/entrypoints.txt +++ b/libc/config/baremetal/riscv/entrypoints.txt @@ -207,6 +207,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.acoshf libc.src.math.asinf libc.src.math.asinhf + libc.src.math.atan2f libc.src.math.atanf libc.src.math.atanhf libc.src.math.ceil diff --git a/libc/config/darwin/arm/entrypoints.txt b/libc/config/darwin/arm/entrypoints.txt index 02a092566069..6b89ce55d72b 100644 --- a/libc/config/darwin/arm/entrypoints.txt +++ b/libc/config/darwin/arm/entrypoints.txt @@ -118,6 +118,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.acoshf libc.src.math.asinf libc.src.math.asinhf + libc.src.math.atan2f libc.src.math.atanf libc.src.math.atanhf libc.src.math.copysign diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index 78da7f0b334b..4ba2d8387b07 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -330,6 +330,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.acoshf libc.src.math.asinf libc.src.math.asinhf + libc.src.math.atan2f libc.src.math.atanf libc.src.math.atanhf libc.src.math.copysign diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt index 6e63e270280e..04baa4c1cf93 100644 --- a/libc/config/linux/arm/entrypoints.txt +++ b/libc/config/linux/arm/entrypoints.txt @@ -198,6 +198,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.acoshf libc.src.math.asinf libc.src.math.asinhf + libc.src.math.atan2f libc.src.math.atanf libc.src.math.atanhf libc.src.math.ceil diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt index 5aae4e246cfb..25745513b920 100644 --- a/libc/config/linux/riscv/entrypoints.txt +++ b/libc/config/linux/riscv/entrypoints.txt @@ -338,6 +338,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.acoshf libc.src.math.asinf libc.src.math.asinhf + libc.src.math.atan2f libc.src.math.atanf libc.src.math.atanhf libc.src.math.copysign diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index a4d0da5e043d..c1d2bfa848df 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -348,6 +348,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.acoshf libc.src.math.asinf libc.src.math.asinhf + libc.src.math.atan2f libc.src.math.atanf libc.src.math.atanhf libc.src.math.canonicalize diff --git a/libc/config/windows/entrypoints.txt b/libc/config/windows/entrypoints.txt index f4456f561ec0..c38125a64622 100644 --- a/libc/config/windows/entrypoints.txt +++ b/libc/config/windows/entrypoints.txt @@ -116,6 +116,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.acoshf libc.src.math.asinf libc.src.math.asinhf + libc.src.math.atan2f libc.src.math.atanf libc.src.math.atanhf libc.src.math.copysign diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst index 080b6a4427f5..b7f1b8739648 100644 --- a/libc/docs/math/index.rst +++ b/libc/docs/math/index.rst @@ -422,7 +422,7 @@ Higher Math Functions +------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | atan2 | | | | | | | | | | | | | +------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| atan2f | | | | | | | | | | | | | +| atan2f | |check| | |check| | |check| | |check| | |check| | | | |check| | |check| | |check| | | | +------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | atan2l | | | | | | | | | | | | | +------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ @@ -591,6 +591,7 @@ acosh |check| asin |check| asinh |check| atan |check| +atan2 |check| atanh |check| cos |check| large cosh |check| diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index ac6e1d1801ba..719bb9aa18cb 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -620,10 +620,14 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"tanhf", RetValSpec, [ArgSpec]>, FunctionSpec<"acosf", RetValSpec, [ArgSpec]>, + FunctionSpec<"asinf", RetValSpec, [ArgSpec]>, FunctionSpec<"asin", RetValSpec, [ArgSpec]>, + FunctionSpec<"atanf", RetValSpec, [ArgSpec]>, + FunctionSpec<"atan2f", RetValSpec, [ArgSpec, ArgSpec]>, + FunctionSpec<"acoshf", RetValSpec, [ArgSpec]>, FunctionSpec<"asinhf", RetValSpec, [ArgSpec]>, FunctionSpec<"atanhf", RetValSpec, [ArgSpec]>, diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt index 4d9b91150d02..b164d33e204b 100644 --- a/libc/src/math/generic/CMakeLists.txt +++ b/libc/src/math/generic/CMakeLists.txt @@ -2854,6 +2854,24 @@ add_entrypoint_object( -O3 ) +add_entrypoint_object( + atan2f + SRCS + atan2f.cpp + HDRS + ../atan2f.h + COMPILE_OPTIONS + -O3 + DEPENDS + .inv_trigf_utils + libc.src.__support.FPUtil.fp_bits + libc.src.__support.FPUtil.multiply_add + libc.src.__support.FPUtil.nearest_integer + libc.src.__support.FPUtil.polyeval + libc.src.__support.FPUtil.rounding_mode + libc.src.__support.macros.optimization +) + add_entrypoint_object( scalbn SRCS diff --git a/libc/src/math/generic/atan2f.cpp b/libc/src/math/generic/atan2f.cpp new file mode 100644 index 000000000000..b79410dbf66e --- /dev/null +++ b/libc/src/math/generic/atan2f.cpp @@ -0,0 +1,306 @@ +//===-- Single-precision atan2f function ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/atan2f.h" +#include "inv_trigf_utils.h" +#include "src/__support/FPUtil/FPBits.h" +#include "src/__support/FPUtil/PolyEval.h" +#include "src/__support/FPUtil/double_double.h" +#include "src/__support/FPUtil/multiply_add.h" +#include "src/__support/FPUtil/nearest_integer.h" +#include "src/__support/FPUtil/rounding_mode.h" +#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY + +namespace LIBC_NAMESPACE { + +namespace { + +// Look up tables for accurate pass: + +// atan(i/16) with i = 0..16, generated by Sollya with: +// > for i from 0 to 16 do { +// a = round(atan(i/16), D, RN); +// b = round(atan(i/16) - a, D, RN); +// print("{", b, ",", a, "},"); +// }; +constexpr fputil::DoubleDouble ATAN_I[17] = { + {0.0, 0.0}, + {-0x1.c934d86d23f1dp-60, 0x1.ff55bb72cfdeap-5}, + {-0x1.cd37686760c17p-59, 0x1.fd5ba9aac2f6ep-4}, + {0x1.347b0b4f881cap-58, 0x1.7b97b4bce5b02p-3}, + {0x1.8ab6e3cf7afbdp-57, 0x1.f5b75f92c80ddp-3}, + {-0x1.963a544b672d8p-57, 0x1.362773707ebccp-2}, + {-0x1.c63aae6f6e918p-56, 0x1.6f61941e4def1p-2}, + {-0x1.24dec1b50b7ffp-56, 0x1.a64eec3cc23fdp-2}, + {0x1.a2b7f222f65e2p-56, 0x1.dac670561bb4fp-2}, + {-0x1.d5b495f6349e6p-56, 0x1.0657e94db30dp-1}, + {-0x1.928df287a668fp-58, 0x1.1e00babdefeb4p-1}, + {0x1.1021137c71102p-55, 0x1.345f01cce37bbp-1}, + {0x1.2419a87f2a458p-56, 0x1.4978fa3269ee1p-1}, + {0x1.0028e4bc5e7cap-57, 0x1.5d58987169b18p-1}, + {-0x1.8c34d25aadef6p-56, 0x1.700a7c5784634p-1}, + {-0x1.bf76229d3b917p-56, 0x1.819d0b7158a4dp-1}, + {0x1.1a62633145c07p-55, 0x1.921fb54442d18p-1}, +}; + +// Taylor polynomial, generated by Sollya with: +// > for i from 0 to 8 do { +// j = (-1)^(i + 1)/(2*i + 1); +// a = round(j, D, RN); +// b = round(j - a, D, RN); +// print("{", b, ",", a, "},"); +// }; +constexpr fputil::DoubleDouble COEFFS[9] = { + {0.0, 1.0}, // 1 + {-0x1.5555555555555p-56, -0x1.5555555555555p-2}, // -1/3 + {-0x1.999999999999ap-57, 0x1.999999999999ap-3}, // 1/5 + {-0x1.2492492492492p-57, -0x1.2492492492492p-3}, // -1/7 + {0x1.c71c71c71c71cp-58, 0x1.c71c71c71c71cp-4}, // 1/9 + {0x1.745d1745d1746p-59, -0x1.745d1745d1746p-4}, // -1/11 + {-0x1.3b13b13b13b14p-58, 0x1.3b13b13b13b14p-4}, // 1/13 + {-0x1.1111111111111p-60, -0x1.1111111111111p-4}, // -1/15 + {0x1.e1e1e1e1e1e1ep-61, 0x1.e1e1e1e1e1e1ep-5}, // 1/17 +}; + +// Veltkamp's splitting of a double precision into hi + lo, where the hi part is +// slightly smaller than an even split, so that the product of +// hi * (s1 * k + s2) is exact, +// where: +// s1, s2 are single precsion, +// 1/16 <= s1/s2 <= 1 +// 1/16 <= k <= 1 is an integer. +// So the maximal precision of (s1 * k + s2) is: +// prec(s1 * k + s2) = 2 + log2(msb(s2)) - log2(lsb(k_d * s1)) +// = 2 + log2(msb(s1)) + 4 - log2(lsb(k_d)) - log2(lsb(s1)) +// = 2 + log2(lsb(s1)) + 23 + 4 - (-4) - log2(lsb(s1)) +// = 33. +// Thus, the Veltkamp splitting constant is C = 2^33 + 1. +// This is used when FMA instruction is not available. +[[maybe_unused]] constexpr fputil::DoubleDouble split_d(double a) { + fputil::DoubleDouble r{0.0, 0.0}; + constexpr double C = 0x1.0p33 + 1.0; + double t1 = C * a; + double t2 = a - t1; + r.hi = t1 + t2; + r.lo = a - r.hi; + return r; +} + +// Compute atan( num_d / den_d ) in double-double precision. +// num_d = min(|x|, |y|) +// den_d = max(|x|, |y|) +// q_d = num_d / den_d +// idx, k_d = round( 2^4 * num_d / den_d ) +// final_sign = sign of the final result +// const_term = the constant term in the final expression. +float atan2f_double_double(double num_d, double den_d, double q_d, int idx, + double k_d, double final_sign, + const fputil::DoubleDouble &const_term) { + fputil::DoubleDouble q; + double num_r, den_r; + + if (idx != 0) { + // The following range reduction is accurate even without fma for + // 1/16 <= n/d <= 1. + // atan(n/d) - atan(idx/16) = atan((n/d - idx/16) / (1 + (n/d) * (idx/16))) + // = atan((n - d*(idx/16)) / (d + n*idx/16)) + k_d *= 0x1.0p-4; + num_r = fputil::multiply_add(k_d, -den_d, num_d); // Exact + den_r = fputil::multiply_add(k_d, num_d, den_d); // Exact + q.hi = num_r / den_r; + } else { + // For 0 < n/d < 1/16, we just need to calculate the lower part of their + // quotient. + q.hi = q_d; + num_r = num_d; + den_r = den_d; + } +#ifdef LIBC_TARGET_CPU_HAS_FMA + q.lo = fputil::multiply_add(q.hi, -den_r, num_r) / den_r; +#else + // Compute `(num_r - q.hi * den_r) / den_r` accurately without FMA + // instructions. + fputil::DoubleDouble q_hi_dd = split_d(q.hi); + double t1 = fputil::multiply_add(q_hi_dd.hi, -den_r, num_r); // Exact + double t2 = fputil::multiply_add(q_hi_dd.lo, -den_r, t1); + q.lo = t2 / den_r; +#endif // LIBC_TARGET_CPU_HAS_FMA + + // Taylor polynomial, evaluating using Horner's scheme: + // P = x - x^3/3 + x^5/5 -x^7/7 + x^9/9 - x^11/11 + x^13/13 - x^15/15 + // + x^17/17 + // = x*(1 + x^2*(-1/3 + x^2*(1/5 + x^2*(-1/7 + x^2*(1/9 + x^2* + // *(-1/11 + x^2*(1/13 + x^2*(-1/15 + x^2 * 1/17)))))))) + fputil::DoubleDouble q2 = fputil::quick_mult(q, q); + fputil::DoubleDouble p_dd = + fputil::polyeval(q2, COEFFS[0], COEFFS[1], COEFFS[2], COEFFS[3], + COEFFS[4], COEFFS[5], COEFFS[6], COEFFS[7], COEFFS[8]); + fputil::DoubleDouble r_dd = + fputil::add(const_term, fputil::multiply_add(q, p_dd, ATAN_I[idx])); + r_dd.hi *= final_sign; + r_dd.lo *= final_sign; + + // Make sure the sum is normalized: + fputil::DoubleDouble rr = fputil::exact_add(r_dd.hi, r_dd.lo); + // Round to odd. + uint64_t rr_bits = cpp::bit_cast(rr.hi); + if (LIBC_UNLIKELY(((rr_bits & 0xfff'ffff) == 0) && (rr.lo != 0.0))) { + Sign hi_sign = fputil::FPBits(rr.hi).sign(); + Sign lo_sign = fputil::FPBits(rr.lo).sign(); + if (hi_sign == lo_sign) { + ++rr_bits; + } else if ((rr_bits & fputil::FPBits::FRACTION_MASK) > 0) { + --rr_bits; + } + } + + return static_cast(cpp::bit_cast(rr_bits)); +} + +} // anonymous namespace + +// There are several range reduction steps we can take for atan2(y, x) as +// follow: + +// * Range reduction 1: signness +// atan2(y, x) will return a number between -PI and PI representing the angle +// forming by the 0x axis and the vector (x, y) on the 0xy-plane. +// In particular, we have that: +// atan2(y, x) = atan( y/x ) if x >= 0 and y >= 0 (I-quadrant) +// = pi + atan( y/x ) if x < 0 and y >= 0 (II-quadrant) +// = -pi + atan( y/x ) if x < 0 and y < 0 (III-quadrant) +// = atan( y/x ) if x >= 0 and y < 0 (IV-quadrant) +// Since atan function is odd, we can use the formula: +// atan(-u) = -atan(u) +// to adjust the above conditions a bit further: +// atan2(y, x) = atan( |y|/|x| ) if x >= 0 and y >= 0 (I-quadrant) +// = pi - atan( |y|/|x| ) if x < 0 and y >= 0 (II-quadrant) +// = -pi + atan( |y|/|x| ) if x < 0 and y < 0 (III-quadrant) +// = -atan( |y|/|x| ) if x >= 0 and y < 0 (IV-quadrant) +// Which can be simplified to: +// atan2(y, x) = sign(y) * atan( |y|/|x| ) if x >= 0 +// = sign(y) * (pi - atan( |y|/|x| )) if x < 0 + +// * Range reduction 2: reciprocal +// Now that the argument inside atan is positive, we can use the formula: +// atan(1/x) = pi/2 - atan(x) +// to make the argument inside atan <= 1 as follow: +// atan2(y, x) = sign(y) * atan( |y|/|x|) if 0 <= |y| <= x +// = sign(y) * (pi/2 - atan( |x|/|y| ) if 0 <= x < |y| +// = sign(y) * (pi - atan( |y|/|x| )) if 0 <= |y| <= -x +// = sign(y) * (pi/2 + atan( |x|/|y| )) if 0 <= -x < |y| + +// * Range reduction 3: look up table. +// After the previous two range reduction steps, we reduce the problem to +// compute atan(u) with 0 <= u <= 1, or to be precise: +// atan( n / d ) where n = min(|x|, |y|) and d = max(|x|, |y|). +// An accurate polynomial approximation for the whole [0, 1] input range will +// require a very large degree. To make it more efficient, we reduce the input +// range further by finding an integer idx such that: +// | n/d - idx/16 | <= 1/32. +// In particular, +// idx := 2^-4 * round(2^4 * n/d) +// Then for the fast pass, we find a polynomial approximation for: +// atan( n/d ) ~ atan( idx/16 ) + (n/d - idx/16) * Q(n/d - idx/16) +// For the accurate pass, we use the addition formula: +// atan( n/d ) - atan( idx/16 ) = atan( (n/d - idx/16)/(1 + (n*idx)/(16*d)) ) +// = atan( (n - d * idx/16)/(d + n * idx/16) ) +// And finally we use Taylor polynomial to compute the RHS in the accurate pass: +// atan(u) ~ P(u) = u - u^3/3 + u^5/5 - u^7/7 + u^9/9 - u^11/11 + u^13/13 - +// - u^15/15 + u^17/17 +// It's error in double-double precision is estimated in Sollya to be: +// > P = x - x^3/3 + x^5/5 -x^7/7 + x^9/9 - x^11/11 + x^13/13 - x^15/15 +// + x^17/17; +// > dirtyinfnorm(atan(x) - P, [-2^-5, 2^-5]); +// 0x1.aec6f...p-100 +// which is about rounding errors of double-double (2^-104). + +LLVM_LIBC_FUNCTION(float, atan2f, (float y, float x)) { + using FPBits = typename fputil::FPBits; + constexpr double IS_NEG[2] = {1.0, -1.0}; + constexpr double PI = 0x1.921fb54442d18p1; + constexpr double PI_LO = 0x1.1a62633145c07p-53; + constexpr double PI_OVER_4 = 0x1.921fb54442d18p-1; + constexpr double PI_OVER_2 = 0x1.921fb54442d18p0; + constexpr double THREE_PI_OVER_4 = 0x1.2d97c7f3321d2p+1; + // Adjustment for constant term: + // CONST_ADJ[x_sign][y_sign][recip] + constexpr fputil::DoubleDouble CONST_ADJ[2][2][2] = { + {{{0.0, 0.0}, {-PI_LO / 2, -PI_OVER_2}}, + {{-0.0, -0.0}, {-PI_LO / 2, -PI_OVER_2}}}, + {{{-PI_LO, -PI}, {PI_LO / 2, PI_OVER_2}}, + {{-PI_LO, -PI}, {PI_LO / 2, PI_OVER_2}}}}; + + FPBits x_bits(x), y_bits(y); + bool x_sign = x_bits.sign().is_neg(); + bool y_sign = y_bits.sign().is_neg(); + x_bits.set_sign(Sign::POS); + y_bits.set_sign(Sign::POS); + uint32_t x_abs = x_bits.uintval(); + uint32_t y_abs = y_bits.uintval(); + uint32_t max_abs = x_abs > y_abs ? x_abs : y_abs; + uint32_t min_abs = x_abs <= y_abs ? x_abs : y_abs; + + if (LIBC_UNLIKELY(max_abs >= 0x7f80'0000U || min_abs == 0U)) { + if (x_bits.is_nan() || y_bits.is_nan()) + return FPBits::quiet_nan().get_val(); + size_t x_except = x_abs == 0 ? 0 : (x_abs == 0x7f80'0000 ? 2 : 1); + size_t y_except = y_abs == 0 ? 0 : (y_abs == 0x7f80'0000 ? 2 : 1); + + // Exceptional cases: + // EXCEPT[y_except][x_except][x_is_neg] + // with x_except & y_except: + // 0: zero + // 1: finite, non-zero + // 2: infinity + constexpr double EXCEPTS[3][3][2] = { + {{0.0, PI}, {0.0, PI}, {0.0, PI}}, + {{PI_OVER_2, PI_OVER_2}, {0.0, 0.0}, {0.0, PI}}, + {{PI_OVER_2, PI_OVER_2}, + {PI_OVER_2, PI_OVER_2}, + {PI_OVER_4, THREE_PI_OVER_4}}, + }; + + double r = IS_NEG[y_sign] * EXCEPTS[y_except][x_except][x_sign]; + + return static_cast(r); + } + + bool recip = x_abs < y_abs; + double final_sign = IS_NEG[(x_sign != y_sign) != recip]; + fputil::DoubleDouble const_term = CONST_ADJ[x_sign][y_sign][recip]; + double num_d = static_cast(FPBits(min_abs).get_val()); + double den_d = static_cast(FPBits(max_abs).get_val()); + double q_d = num_d / den_d; + + double k_d = fputil::nearest_integer(q_d * 0x1.0p4f); + int idx = static_cast(k_d); + q_d = fputil::multiply_add(k_d, -0x1.0p-4, q_d); + + double p = atan_eval(q_d, idx); + double r = final_sign * + fputil::multiply_add(q_d, p, const_term.hi + ATAN_COEFFS[idx][0]); + + constexpr uint32_t LOWER_ERR = 4; + // Mask sticky bits in double precision before rounding to single precision. + constexpr uint32_t MASK = + mask_trailing_ones::SIG_LEN - + FPBits::SIG_LEN - 1>(); + constexpr uint32_t UPPER_ERR = MASK - LOWER_ERR; + + uint32_t r_bits = static_cast(cpp::bit_cast(r)) & MASK; + + // Ziv's rounding test. + if (LIBC_LIKELY(r_bits > LOWER_ERR && r_bits < UPPER_ERR)) + return static_cast(r); + + return atan2f_double_double(num_d, den_d, q_d, idx, k_d, final_sign, + const_term); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/atanf.cpp b/libc/src/math/generic/atanf.cpp index 4adda429cc04..9fa1a331c9c0 100644 --- a/libc/src/math/generic/atanf.cpp +++ b/libc/src/math/generic/atanf.cpp @@ -60,11 +60,15 @@ LLVM_LIBC_FUNCTION(float, atanf, (float x)) { } // Use Taylor polynomial: // atan(x) ~ x * (1 - x^2 / 3 + x^4 / 5 - x^6 / 7 + x^8 / 9 - x^10 / 11). + constexpr double ATAN_TAYLOR[6] = { + 0x1.0000000000000p+0, -0x1.5555555555555p-2, 0x1.999999999999ap-3, + -0x1.2492492492492p-3, 0x1.c71c71c71c71cp-4, -0x1.745d1745d1746p-4, + }; double x2 = x_d * x_d; double x4 = x2 * x2; - double c0 = fputil::multiply_add(x2, ATAN_COEFFS[0][1], ATAN_COEFFS[0][0]); - double c1 = fputil::multiply_add(x2, ATAN_COEFFS[0][3], ATAN_COEFFS[0][2]); - double c2 = fputil::multiply_add(x2, ATAN_COEFFS[0][5], ATAN_COEFFS[0][4]); + double c0 = fputil::multiply_add(x2, ATAN_TAYLOR[1], ATAN_TAYLOR[0]); + double c1 = fputil::multiply_add(x2, ATAN_TAYLOR[3], ATAN_TAYLOR[2]); + double c2 = fputil::multiply_add(x2, ATAN_TAYLOR[5], ATAN_TAYLOR[4]); double p = fputil::polyeval(x4, c0, c1, c2); double r = fputil::multiply_add(x_d, p, const_term); return static_cast(r); @@ -81,11 +85,6 @@ LLVM_LIBC_FUNCTION(float, atanf, (float x)) { int idx; if (x_abs > 0x3f80'0000U) { - // Exceptional value: - if (LIBC_UNLIKELY(x_abs == 0x3ffe'2ec1U)) { // |x| = 0x1.fc5d82p+0 - return sign.is_pos() ? fputil::round_result_slightly_up(0x1.1ab2fp0f) - : fputil::round_result_slightly_down(-0x1.1ab2fp0f); - } // |x| > 1, we need to invert x, so we will perform range reduction in // double precision. x_d = 1.0 / static_cast(x_bits.get_val()); @@ -98,10 +97,9 @@ LLVM_LIBC_FUNCTION(float, atanf, (float x)) { SIGNED_PI_OVER_2[sign.is_neg()]); } else { // Exceptional value: - if (LIBC_UNLIKELY(x_abs == 0x3dbb'6ac7U)) { // |x| = 0x1.76d58ep-4 - return sign.is_pos() - ? fputil::round_result_slightly_up(0x1.75cb06p-4f) - : fputil::round_result_slightly_down(-0x1.75cb06p-4f); + if (LIBC_UNLIKELY(x_abs == 0x3d8d'6b23U)) { // |x| = 0x1.1ad646p-4 + return sign.is_pos() ? fputil::round_result_slightly_down(0x1.1a6386p-4f) + : fputil::round_result_slightly_up(-0x1.1a6386p-4f); } // Perform range reduction in single precision. float x_f = x_bits.get_val(); diff --git a/libc/src/math/generic/inv_trigf_utils.cpp b/libc/src/math/generic/inv_trigf_utils.cpp index 93d5bcbf7b56..19c8b997dc4e 100644 --- a/libc/src/math/generic/inv_trigf_utils.cpp +++ b/libc/src/math/generic/inv_trigf_utils.cpp @@ -16,65 +16,70 @@ namespace LIBC_NAMESPACE { // Generated by Sollya with: // > for i from 1 to 16 do { // mid_point = i/16; -// P = fpminimax(atan(mid_point + x), 7, [|D...|], [-1/32, 1/32]); +// P = fpminimax(atan(mid_point + x), 8, [|D...|], [-1/32, 1/32]); // print("{", coeff(P, 0), ",", coeff(P, 1), ",", coeff(P, 2), ",", // coeff(P, 3), ",", coeff(P, 4), ",", coeff(P, 5), ",", coeff(P, 6), -// ",", coeff(P, 7), "},"); +// ",", coeff(P, 7), ",", coeff(P, 8), "},"); // }; -// For i = 0, ATAN_COEFFS[0][j] = (-1)^j * (1/(2*j + 1)) is the odd coefficients -// of the Taylor polynomial of atan(x). -double ATAN_COEFFS[17][8] = { - {0x1.0000000000000p+0, -0x1.5555555555555p-2, 0x1.999999999999ap-3, - -0x1.2492492492492p-3, 0x1.c71c71c71c71cp-4, -0x1.745d1745d1746p-4, - 0x1.3b13b13b13b14p-4, -0x1.1111111111111p-4}, - {0x1.ff55bb72cfdb1p-5, 0x1.fe01fe01fe1bp-1, -0x1.fc05f80821d1ap-5, - -0x1.4d6930419fc5fp-2, 0x1.f61b9f6d69313p-5, 0x1.8208a32f4346cp-3, - -0x1.ecb8fc53d04efp-5, -0x1.060710cb59cbcp-3}, - {0x1.fd5ba9aac2f3cp-4, 0x1.f81f81f81f96ap-1, -0x1.f05e09cf4c1b2p-4, - -0x1.368c3aac7543ep-2, 0x1.d9b14bddfac55p-4, 0x1.4048e55ec725ep-3, - -0x1.b98ca3c1594b5p-4, -0x1.664eabaabbc16p-4}, - {0x1.7b97b4bce5ae7p-3, 0x1.ee9c7f8458f06p-1, -0x1.665c226c8dc69p-3, - -0x1.1344bb77961b7p-2, 0x1.42ac97745d3ccp-3, 0x1.c32e142047ec1p-4, - -0x1.137ae41ab96cbp-3, -0x1.1a6ae8c09a4b6p-5}, - {0x1.f5b75f92c80c6p-3, 0x1.e1e1e1e1e1ed4p-1, -0x1.c5894d101ad4p-3, - -0x1.ce6de02b38c38p-3, 0x1.78a3920c336b9p-3, 0x1.dd5ff94a9d499p-5, - -0x1.1ac2d3f9d072ep-3, 0x1.0af9735dff373p-6}, - {0x1.362773707ebc5p-2, 0x1.d272ca3fc5b8bp-1, -0x1.0997e8ae90cb6p-2, - -0x1.6cf6667146798p-3, 0x1.8dd1dff17f3d3p-3, 0x1.24860eced656fp-7, - -0x1.f4220e8f18ed5p-4, 0x1.b700aed7cdc34p-5}, - {0x1.6f61941e4deeep-2, 0x1.c0e070381c115p-1, -0x1.2726dd1347c7ep-2, - -0x1.09f37b3ad010dp-3, 0x1.85eaca5196f5cp-3, -0x1.04d640117852ap-5, - -0x1.802c2956871c7p-4, 0x1.2992b45df0ee7p-4}, - {0x1.a64eec3cc23fep-2, 0x1.adbe87f94906bp-1, -0x1.3b9d8eab5eae5p-2, - -0x1.57c09646faabbp-4, 0x1.6795330e73aep-3, -0x1.f2d89a702a652p-5, - -0x1.f3afd90a9d4d7p-5, 0x1.3261723d3f153p-4}, - {0x1.dac670561bb53p-2, 0x1.999999999998fp-1, -0x1.47ae147afd8cap-2, - -0x1.5d867c3dfd72ap-5, 0x1.3a92a76cba833p-3, -0x1.3ec460286928ap-4, - -0x1.ed02ff86892acp-6, 0x1.0a674c8f05727p-4}, - {0x1.0657e94db30d2p-1, 0x1.84f00c27805ffp-1, -0x1.4c62cb564f677p-2, - -0x1.e6495b262dfe7p-8, 0x1.063c34eca262bp-3, -0x1.58b78dc79b5aep-4, - -0x1.4623815233be1p-8, 0x1.93afe94328089p-5}, - {0x1.1e00babdefeb6p-1, 0x1.702e05c0b8159p-1, -0x1.4af2b78236bd6p-2, - 0x1.5d0b7ea46ed08p-6, 0x1.a124870236935p-4, -0x1.519e1ec133a88p-4, - 0x1.a54632a3f48c7p-7, 0x1.099ca0945096dp-5}, - {0x1.345f01cce37bdp-1, 0x1.5babcc647fa7ep-1, -0x1.449db09443a67p-2, - 0x1.655caac78a0fcp-5, 0x1.3bbbdb0d09efap-4, -0x1.34a306c27e021p-4, - 0x1.83fe749c7966p-6, 0x1.2057cc96d9edcp-6}, - {0x1.4978fa3269ee2p-1, 0x1.47ae147ae146bp-1, -0x1.3a92a305652e1p-2, - 0x1.ec21b5172657fp-5, 0x1.c2f8c45d2f4eep-5, -0x1.0ba99c4aeb8acp-4, - 0x1.d716a4af4d1d6p-6, 0x1.97fba0a9696dep-8}, - {0x1.5d58987169b19p-1, 0x1.34679ace0133cp-1, -0x1.2ddfb03920e2fp-2, - 0x1.2491307c0fa0bp-4, 0x1.29c7eca0136fp-5, -0x1.bca792caa6f1cp-5, - 0x1.e5d92545576bcp-6, -0x1.8ca76fcf5ccd2p-10}, - {0x1.700a7c5784634p-1, 0x1.21fb78121fb71p-1, -0x1.1f6a8499ea541p-2, - 0x1.41b15e5e77bcfp-4, 0x1.59bc9bf54fb02p-6, -0x1.63b54ff058e0fp-5, - 0x1.c8da01221306fp-6, -0x1.906b2c274c39cp-8}, - {0x1.819d0b7158a4dp-1, 0x1.107fbbe01107cp-1, -0x1.0feeb40897d4ep-2, - 0x1.50e5afb95f5d6p-4, 0x1.2a7c2f0c7495dp-7, -0x1.12bd2bb5062cdp-5, - 0x1.93e8ceb89afebp-6, -0x1.10da9b8c6b731p-7}, - {0x1.921fb54442d18p-1, 0x1.fffffffffffebp-2, -0x1.fffffffffcbbcp-3, - 0x1.555555564e2fep-4, -0x1.20b17d5dd89dcp-30, -0x1.9999c5ad71711p-6, - 0x1.5558b76e7aaf9p-6, -0x1.236e803c6c1f6p-7}, +// For i = 0, the polynomial is generated by: +// > P = fpminimax(atan(x)/x, 7, [|1, D...|], [0, 1/32]); +// > dirtyinfnorm((atan(x) - x*P)/x, [0, 1/32]); +// 0x1.feb2fcdba66447ccbe28a1a0f935b51678a718fb1p-59 +// Notice that degree-7 is good enough for atanf, but degree-8 helps reduce the +// error bounds for atan2f's fast pass 16 times, and it does not affect the +// performance of atanf much. +double ATAN_COEFFS[17][9] = { + {0.0, 1.0, 0x1.3f8d76d26d61bp-47, -0x1.5555555574cd8p-2, + 0x1.0dde5d06878eap-29, 0x1.99997738acc77p-3, 0x1.2c43eac9797cap-16, + -0x1.25fb020007dbdp-3, 0x1.c1b6c31d7b0aep-7}, + {0x1.ff55bb72cfde9p-5, 0x1.fe01fe01fe007p-1, -0x1.fc05f809ed8dap-5, + -0x1.4d69303afe04ep-2, 0x1.f61bc3e8349cp-5, 0x1.820839278756bp-3, + -0x1.eda4de1c6bf3fp-5, -0x1.0514d42d64a63p-3, 0x1.db3746a442dcbp-5}, + {0x1.fd5ba9aac2f6ep-4, 0x1.f81f81f81f813p-1, -0x1.f05e09d0dc378p-4, + -0x1.368c3aa719215p-2, 0x1.d9b16b33ff9c9p-4, 0x1.40488f9c6262ap-3, + -0x1.ba55933e62ea5p-4, -0x1.64c6a15cd9116p-4, 0x1.9273d5939a75ap-4}, + {0x1.7b97b4bce5b02p-3, 0x1.ee9c7f8458e05p-1, -0x1.665c226d6961p-3, + -0x1.1344bb7391703p-2, 0x1.42aca8b0081b9p-3, 0x1.c32d9381d7c03p-4, + -0x1.13e970672e246p-3, -0x1.181ed934dd733p-5, 0x1.bad81ea190c08p-4}, + {0x1.f5b75f92c80ddp-3, 0x1.e1e1e1e1e1e2cp-1, -0x1.c5894d10d363dp-3, + -0x1.ce6de025f9f5ep-3, 0x1.78a3a07c8dd7fp-3, 0x1.dd5f5180f386ep-5, + -0x1.1b1f513c4536bp-3, 0x1.0df852e58c43cp-6, 0x1.722e7a7e42505p-4}, + {0x1.362773707ebccp-2, 0x1.d272ca3fc5b2ep-1, -0x1.0997e8aeca8fbp-2, + -0x1.6cf6666e5e693p-3, 0x1.8dd1e907e88adp-3, 0x1.24849ac0caa5dp-7, + -0x1.f496be486229dp-4, 0x1.b7d54b8e759ecp-5, 0x1.d39c0d39c3922p-5}, + {0x1.6f61941e4def1p-2, 0x1.c0e070381c0f2p-1, -0x1.2726dd135d9eep-2, + -0x1.09f37b39b70e4p-3, 0x1.85eacdaadd712p-3, -0x1.04d66340d5b9p-5, + -0x1.8056b15a22b98p-4, 0x1.29baf494ad3ddp-4, 0x1.52d5881322a7ap-6}, + {0x1.a64eec3cc23fdp-2, 0x1.adbe87f94906ap-1, -0x1.3b9d8eab55addp-2, + -0x1.57c09646eb7p-4, 0x1.6795319e3b8dfp-3, -0x1.f2d89b5ef31bep-5, + -0x1.f38aac26203cap-5, 0x1.3262802235e3fp-4, -0x1.2afd6b9a57d66p-7}, + {0x1.dac670561bb4fp-2, 0x1.99999999999ap-1, -0x1.47ae147adff11p-2, + -0x1.5d867c40188b7p-5, 0x1.3a92a2df85e7ap-3, -0x1.3ec457c46e851p-4, + -0x1.ec1b9777e2e5bp-6, 0x1.0a542992a821ep-4, -0x1.ccffbe2f0d945p-6}, + {0x1.0657e94db30dp-1, 0x1.84f00c2780615p-1, -0x1.4c62cb562defap-2, + -0x1.e6495b3c14e03p-8, 0x1.063c2fa617bfcp-3, -0x1.58b782d9907aap-4, + -0x1.41e6ff524b7fp-8, 0x1.937dfff3205a7p-5, -0x1.0fb1fd1c729dp-5}, + {0x1.1e00babdefeb4p-1, 0x1.702e05c0b816ep-1, -0x1.4af2b78215fbep-2, + 0x1.5d0b7e9f36997p-6, 0x1.a1247cb978debp-4, -0x1.519e1457734cap-4, + 0x1.a755cf86b5bfbp-7, 0x1.096d174284564p-5, -0x1.081adf539ad58p-5}, + {0x1.345f01cce37bbp-1, 0x1.5babcc647fa8ep-1, -0x1.449db09426a6dp-2, + 0x1.655caac5896dap-5, 0x1.3bbbd22d05a61p-4, -0x1.34a2febee042fp-4, + 0x1.84df9c8269e34p-6, 0x1.200e8176c899ap-6, -0x1.c00b23c3ce222p-6}, + {0x1.4978fa3269ee1p-1, 0x1.47ae147ae1477p-1, -0x1.3a92a3055231ap-2, + 0x1.ec21b515a4a2p-5, 0x1.c2f8b81f9a0d2p-5, -0x1.0ba9964125453p-4, + 0x1.d7b5614777a05p-6, 0x1.971e91ed73595p-8, -0x1.3fc375a78dc74p-6}, + {0x1.5d58987169b18p-1, 0x1.34679ace01343p-1, -0x1.2ddfb039136e5p-2, + 0x1.2491307b9fb73p-4, 0x1.29c7e4886dc22p-5, -0x1.bca78bcca83ap-5, + 0x1.e63efd7cbe1ddp-6, -0x1.8ea6c4f03b42dp-10, -0x1.9385b5c3a6997p-7}, + {0x1.700a7c5784634p-1, 0x1.21fb78121fb76p-1, -0x1.1f6a8499e5d1ap-2, + 0x1.41b15e5e29423p-4, 0x1.59bc953163345p-6, -0x1.63b54b13184ddp-5, + 0x1.c9086666d213p-6, -0x1.90c3b4ad8d4bcp-8, -0x1.80f08ed9f6f57p-8}, + {0x1.819d0b7158a4dp-1, 0x1.107fbbe01107ep-1, -0x1.0feeb4089670ep-2, + 0x1.50e5afb93f5cbp-4, 0x1.2a7c2adffeffbp-7, -0x1.12bd29b4f1b43p-5, + 0x1.93f71f0eb00eap-6, -0x1.10ece5ad30e28p-7, -0x1.db1a76bcd2b9cp-10}, + {0x1.921fb54442d18p-1, 0x1.ffffffffffffep-2, -0x1.fffffffffc51cp-3, + 0x1.555555557002ep-4, -0x1.a88260c338e75p-30, -0x1.99999f9a7614fp-6, + 0x1.555e31a1e15e9p-6, -0x1.245240d65e629p-7, -0x1.fa9ba66478903p-11}, }; } // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/inv_trigf_utils.h b/libc/src/math/generic/inv_trigf_utils.h index c621f27e1014..e60c367d7b46 100644 --- a/libc/src/math/generic/inv_trigf_utils.h +++ b/libc/src/math/generic/inv_trigf_utils.h @@ -19,9 +19,9 @@ namespace LIBC_NAMESPACE { constexpr double M_MATH_PI = 0x1.921fb54442d18p+1; constexpr double M_MATH_PI_2 = 0x1.921fb54442d18p+0; -extern double ATAN_COEFFS[17][8]; +extern double ATAN_COEFFS[17][9]; -// For |x| <= 1/32 and 1 <= i <= 16, return Q(x) such that: +// For |x| <= 1/32 and 0 <= i <= 16, return Q(x) such that: // Q(x) ~ (atan(x + i/16) - atan(i/16)) / x. LIBC_INLINE double atan_eval(double x, int i) { double x2 = x * x; @@ -29,10 +29,11 @@ LIBC_INLINE double atan_eval(double x, int i) { double c0 = fputil::multiply_add(x, ATAN_COEFFS[i][2], ATAN_COEFFS[i][1]); double c1 = fputil::multiply_add(x, ATAN_COEFFS[i][4], ATAN_COEFFS[i][3]); double c2 = fputil::multiply_add(x, ATAN_COEFFS[i][6], ATAN_COEFFS[i][5]); + double c3 = fputil::multiply_add(x, ATAN_COEFFS[i][8], ATAN_COEFFS[i][7]); double x4 = x2 * x2; double d1 = fputil::multiply_add(x2, c1, c0); - double d2 = fputil::multiply_add(x2, ATAN_COEFFS[i][7], c2); + double d2 = fputil::multiply_add(x2, c3, c2); double p = fputil::multiply_add(x4, d2, d1); return p; } diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt index 026bcd12928b..f8f0f8ba7b6f 100644 --- a/libc/test/src/math/CMakeLists.txt +++ b/libc/test/src/math/CMakeLists.txt @@ -1719,6 +1719,19 @@ add_fp_unittest( libc.src.__support.FPUtil.fp_bits ) +add_fp_unittest( + atan2f_test + NEED_MPFR + SUITE + libc-math-unittests + SRCS + atan2f_test.cpp + DEPENDS + libc.include.math + libc.src.math.atan2f + libc.src.__support.FPUtil.fp_bits +) + add_subdirectory(generic) add_subdirectory(smoke) diff --git a/libc/test/src/math/atan2f_test.cpp b/libc/test/src/math/atan2f_test.cpp new file mode 100644 index 000000000000..343e7601b039 --- /dev/null +++ b/libc/test/src/math/atan2f_test.cpp @@ -0,0 +1,133 @@ +//===-- Unittests for atan2f ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "include/llvm-libc-macros/math-macros.h" +#include "src/__support/FPUtil/FPBits.h" +#include "src/math/atan2f.h" +#include "test/UnitTest/FPMatcher.h" +#include "test/UnitTest/Test.h" +#include "utils/MPFRWrapper/MPFRUtils.h" + +using LlvmLibcAtan2fTest = LIBC_NAMESPACE::testing::FPTest; +using LIBC_NAMESPACE::testing::tlog; + +namespace mpfr = LIBC_NAMESPACE::testing::mpfr; + +TEST_F(LlvmLibcAtan2fTest, TrickyInputs) { + constexpr int N = 17; + mpfr::BinaryInput INPUTS[N] = { + {0x1.0cb3a4p+20f, 0x1.4ebacp+22f}, {0x1.12215p+1f, 0x1.4fabfcp+22f}, + {-0x1.13baaep+41f, 0x1.5bd22ep+23f}, {0x1.1ff7dcp+41f, 0x1.aec0a6p+23f}, + {0x1.2bc794p+23f, 0x1.0bc0c6p+23f}, {0x1.2fba3ap+42f, 0x1.f99456p+23f}, + {0x1.5ea1f8p+27f, 0x1.f2a1aep+23f}, {0x1.7a931p+44f, 0x1.352ac4p+22f}, + {0x1.8802bcp+21f, 0x1.8f130ap+23f}, {0x1.658ef8p+17f, 0x1.3c00f4p+22f}, + {0x1.69fb0cp+21f, 0x1.39e4c4p+23f}, {0x1.8eb24cp+11f, 0x1.36518p+23f}, + {0x1.9e7ebp+30f, 0x1.d80522p+23f}, {0x1.b4bdeep+19f, 0x1.c19b4p+23f}, + {0x1.bc201p+43f, 0x1.617346p+23f}, {0x1.c96c3cp+20f, 0x1.c01d1ep+23f}, + {0x1.781fcp+28f, 0x1.dcb3cap+23f}, + }; + + for (int i = 0; i < N; ++i) { + float x = INPUTS[i].x; + float y = INPUTS[i].y; + ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Atan2, INPUTS[i], + LIBC_NAMESPACE::atan2f(x, y), 0.5); + INPUTS[i].x = -INPUTS[i].x; + ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Atan2, INPUTS[i], + LIBC_NAMESPACE::atan2f(-x, y), 0.5); + INPUTS[i].y = -INPUTS[i].y; + ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Atan2, INPUTS[i], + LIBC_NAMESPACE::atan2f(-x, -y), 0.5); + INPUTS[i].x = -INPUTS[i].x; + ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Atan2, INPUTS[i], + LIBC_NAMESPACE::atan2f(x, -y), 0.5); + } +} + +TEST_F(LlvmLibcAtan2fTest, InFloatRange) { + constexpr uint32_t X_COUNT = 1'23; + constexpr uint32_t X_START = FPBits(0.25f).uintval(); + constexpr uint32_t X_STOP = FPBits(4.0f).uintval(); + constexpr uint32_t X_STEP = (X_STOP - X_START) / X_COUNT; + + constexpr uint32_t Y_COUNT = 1'37; + constexpr uint32_t Y_START = FPBits(0.25f).uintval(); + constexpr uint32_t Y_STOP = FPBits(4.0f).uintval(); + constexpr uint32_t Y_STEP = (Y_STOP - Y_START) / Y_COUNT; + + auto test = [&](mpfr::RoundingMode rounding_mode) { + mpfr::ForceRoundingMode __r(rounding_mode); + if (!__r.success) + return; + + uint64_t fails = 0; + uint64_t finite_count = 0; + uint64_t total_count = 0; + float failed_x, failed_y, failed_r = 0.0; + double tol = 0.5; + + for (uint32_t i = 0, v = X_START; i <= X_COUNT; ++i, v += X_STEP) { + float x = FPBits(v).get_val(); + if (isnan(x) || isinf(x) || x < 0.0) + continue; + + for (uint32_t j = 0, w = Y_START; j <= Y_COUNT; ++j, w += Y_STEP) { + float y = FPBits(w).get_val(); + if (isnan(y) || isinf(y)) + continue; + + LIBC_NAMESPACE::libc_errno = 0; + float result = LIBC_NAMESPACE::atan2f(x, y); + ++total_count; + if (isnan(result) || isinf(result)) + continue; + + ++finite_count; + mpfr::BinaryInput inputs{x, y}; + + if (!TEST_MPFR_MATCH_ROUNDING_SILENTLY(mpfr::Operation::Atan2, inputs, + result, 0.5, rounding_mode)) { + ++fails; + while (!TEST_MPFR_MATCH_ROUNDING_SILENTLY( + mpfr::Operation::Atan2, inputs, result, tol, rounding_mode)) { + failed_x = x; + failed_y = y; + failed_r = result; + + if (tol > 1000.0) + break; + + tol *= 2.0; + } + } + } + } + if (fails || (finite_count < total_count)) { + tlog << " Atan2f failed: " << fails << "/" << finite_count << "/" + << total_count << " tests.\n" + << " Max ULPs is at most: " << static_cast(tol) << ".\n"; + } + if (fails) { + mpfr::BinaryInput inputs{failed_x, failed_y}; + EXPECT_MPFR_MATCH(mpfr::Operation::Atan2, inputs, failed_r, 0.5, + rounding_mode); + } + }; + + tlog << " Test Rounding To Nearest...\n"; + test(mpfr::RoundingMode::Nearest); + + tlog << " Test Rounding Downward...\n"; + test(mpfr::RoundingMode::Downward); + + tlog << " Test Rounding Upward...\n"; + test(mpfr::RoundingMode::Upward); + + tlog << " Test Rounding Toward Zero...\n"; + test(mpfr::RoundingMode::TowardZero); +} diff --git a/libc/test/src/math/atanf_test.cpp b/libc/test/src/math/atanf_test.cpp index e51932fc4955..58b0eadd63f8 100644 --- a/libc/test/src/math/atanf_test.cpp +++ b/libc/test/src/math/atanf_test.cpp @@ -55,12 +55,15 @@ TEST_F(LlvmLibcAtanfTest, InFloatRange) { TEST_F(LlvmLibcAtanfTest, SpecialValues) { uint32_t val_arr[] = { 0x3d8d6b23U, // x = 0x1.1ad646p-4f + 0x3dbb6ac7U, // x = 0x1.76d58ep-4f 0x3feefcfbU, // x = 0x1.ddf9f6p+0f + 0x3ffe2ec1U, // x = 0x1.fc5d82p+0f 0xbd8d6b23U, // x = -0x1.1ad646p-4f + 0xbdbb6ac7U, // x = -0x1.76d58ep-4f 0xbfeefcfbU, // x = -0x1.ddf9f6p+0f + 0xbffe2ec1U, // x = -0x1.fc5d82p+0 0x7F800000U, // x = +Inf 0xFF800000U, // x = -Inf - 0xbffe2ec1U, // x = -0x1.fc5d82p+0f }; for (uint32_t v : val_arr) { float x = FPBits(v).get_val(); diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt index ae2cbad7d5a7..5d269ddb229c 100644 --- a/libc/test/src/math/smoke/CMakeLists.txt +++ b/libc/test/src/math/smoke/CMakeLists.txt @@ -2755,6 +2755,18 @@ add_fp_unittest( libc.src.__support.FPUtil.fp_bits ) +add_fp_unittest( + atan2f_test + SUITE + libc-math-smoke-tests + SRCS + atan2f_test.cpp + DEPENDS + libc.src.errno.errno + libc.src.math.atan2f + libc.src.__support.FPUtil.fp_bits +) + add_fp_unittest( scalbn_test SUITE diff --git a/libc/test/src/math/smoke/atan2f_test.cpp b/libc/test/src/math/smoke/atan2f_test.cpp new file mode 100644 index 000000000000..ecac36b3a8c0 --- /dev/null +++ b/libc/test/src/math/smoke/atan2f_test.cpp @@ -0,0 +1,50 @@ +//===-- Unittests for atan2f ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "include/llvm-libc-macros/math-macros.h" +#include "src/__support/FPUtil/FPBits.h" +#include "src/errno/libc_errno.h" +#include "src/math/atan2f.h" +#include "test/UnitTest/FPMatcher.h" +#include "test/UnitTest/Test.h" + +using LlvmLibcAtan2fTest = LIBC_NAMESPACE::testing::FPTest; + +TEST_F(LlvmLibcAtan2fTest, SpecialNumbers) { + LIBC_NAMESPACE::libc_errno = 0; + + LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); + EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::atan2f(aNaN, zero)); + EXPECT_FP_EXCEPTION(0); + EXPECT_MATH_ERRNO(0); + + LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); + EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::atan2f(1.0f, aNaN)); + EXPECT_FP_EXCEPTION(0); + EXPECT_MATH_ERRNO(0); + + LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); + EXPECT_FP_EQ_ALL_ROUNDING(0.0f, LIBC_NAMESPACE::atan2f(zero, zero)); + EXPECT_FP_EXCEPTION(0); + EXPECT_MATH_ERRNO(0); + + LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); + EXPECT_FP_EQ_ALL_ROUNDING(-0.0f, LIBC_NAMESPACE::atan2f(-0.0f, zero)); + EXPECT_FP_EXCEPTION(0); + EXPECT_MATH_ERRNO(0); + + LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); + EXPECT_FP_EQ_ALL_ROUNDING(0.0f, LIBC_NAMESPACE::atan2f(1.0f, inf)); + EXPECT_FP_EXCEPTION(0); + EXPECT_MATH_ERRNO(0); + + LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); + EXPECT_FP_EQ_ALL_ROUNDING(-0.0f, LIBC_NAMESPACE::atan2f(-1.0f, inf)); + EXPECT_FP_EXCEPTION(0); + EXPECT_MATH_ERRNO(0); +} diff --git a/libc/utils/MPFRWrapper/MPFRUtils.cpp b/libc/utils/MPFRWrapper/MPFRUtils.cpp index 2e1c44e6fd5d..a83f7a7ceb92 100644 --- a/libc/utils/MPFRWrapper/MPFRUtils.cpp +++ b/libc/utils/MPFRWrapper/MPFRUtils.cpp @@ -181,6 +181,12 @@ public: return result; } + MPFRNumber atan2(const MPFRNumber &b) { + MPFRNumber result(*this); + mpfr_atan2(result.value, value, b.value, mpfr_rounding); + return result; + } + MPFRNumber atanh() const { MPFRNumber result(*this); mpfr_atanh(result.value, value, mpfr_rounding); @@ -623,6 +629,8 @@ binary_operation_one_output(Operation op, InputType x, InputType y, MPFRNumber inputX(x, precision, rounding); MPFRNumber inputY(y, precision, rounding); switch (op) { + case Operation::Atan2: + return inputX.atan2(inputY); case Operation::Fmod: return inputX.fmod(inputY); case Operation::Hypot: diff --git a/libc/utils/MPFRWrapper/MPFRUtils.h b/libc/utils/MPFRWrapper/MPFRUtils.h index 6164d78fa5ad..d5ff590cd7bb 100644 --- a/libc/utils/MPFRWrapper/MPFRUtils.h +++ b/libc/utils/MPFRWrapper/MPFRUtils.h @@ -68,6 +68,7 @@ enum class Operation : int { // input and produce a single floating point number of the same type as // output. BeginBinaryOperationsSingleOutput, + Atan2, Fmod, Hypot, Pow, -- GitLab From 4746877c2716224dc87c69750bdd0df95b6d5b16 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Mon, 1 Apr 2024 13:51:47 -0400 Subject: [PATCH 0126/1690] [C99] Claim conformance to WG14 N570 --- clang/test/C/C99/n570.c | 31 +++++++++++++++++++++++++++++++ clang/www/c_status.html | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 clang/test/C/C99/n570.c diff --git a/clang/test/C/C99/n570.c b/clang/test/C/C99/n570.c new file mode 100644 index 000000000000..31c09224e618 --- /dev/null +++ b/clang/test/C/C99/n570.c @@ -0,0 +1,31 @@ +// RUN: %clang_cc1 -verify -std=c99 %s +// RUN: %clang_cc1 -E -std=c99 %s | FileCheck %s +// expected-no-diagnostics + +/* WG14 N570: Yes + * Empty macro arguments + * + * NB: the original paper is not available online anywhere, so the test + * coverage is coming from what could be gleaned from the C99 rationale + * document. In C89, it was UB to pass no arguments to a function-like macro, + * and that's now supported in C99. + */ + +#define TEN 10 +#define U u +#define I // expands into no preprocessing tokens +#define L L +#define glue(a, b) a ## b +#define xglue(a, b) glue(a, b) + +const unsigned u = xglue(TEN, U); +const int i = xglue(TEN, I); +const long l = xglue(TEN, L); + +// CHECK: const unsigned u = 10u; +// CHECK-NEXT: const int i = 10; +// CHECK-NEXT: const long l = 10L; + +_Static_assert(u == 10U, ""); +_Static_assert(i == 10, ""); +_Static_assert(l == 10L, ""); diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 028234a8961d..123897593e5d 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -300,7 +300,7 @@ conformance.

empty macro arguments N570 - Unknown + Yes new structure type compatibility (tag compatibility) -- GitLab From 5ff2773d4e606ac57750f1fc2aa4dc49b8dbede1 Mon Sep 17 00:00:00 2001 From: Russell Greene Date: Mon, 1 Apr 2024 12:15:24 -0600 Subject: [PATCH 0127/1690] [clang-cl] Allow a colon after /Fo option (#87209) Modeled after https://github.com/llvm/llvm-project/commit/8513a681f7d8d1188706762e712168aebc3119dd# According to https://learn.microsoft.com/en-us/cpp/build/reference/fo-object-file-name?view=msvc-170, `/Fo` accepts a trailing-colon variant. This is also tested in practice. This allows clang-cl to parse this. I just copied one of the existing tests, let me know if this is not the best way to do this. I tested that the test does not pass beofre the Options.td change, and that it does after. See also #46065 --- clang/include/clang/Driver/Options.td | 1 + clang/test/Driver/cl-outputs.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 04eb87f0d5d1..f5289fb00c89 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -8313,6 +8313,7 @@ def _SLASH_Fi : CLCompileJoined<"Fi">, def _SLASH_Fo : CLCompileJoined<"Fo">, HelpText<"Set output object file (with /c)">, MetaVarName<"">; +def _SLASH_Fo_COLON : CLCompileJoined<"Fo:">, Alias<_SLASH_Fo>; def _SLASH_guard : CLJoined<"guard:">, HelpText<"Enable Control Flow Guard with /guard:cf, or only the table with /guard:cf,nochecks. " "Enable EH Continuation Guard with /guard:ehcont">; diff --git a/clang/test/Driver/cl-outputs.c b/clang/test/Driver/cl-outputs.c index 07ff43642a62..4d58f0fb548b 100644 --- a/clang/test/Driver/cl-outputs.c +++ b/clang/test/Driver/cl-outputs.c @@ -301,5 +301,8 @@ // RUN: %clang_cl -fdebug-compilation-dir=. /Z7 /Foa.obj -### -- %s 2>&1 | FileCheck -check-prefix=RELATIVE_OBJPATH1 %s // RELATIVE_OBJPATH1: "-object-file-name=a.obj" +// RUN: %clang_cl -fdebug-compilation-dir=. /Z7 /Fo:a.obj -### -- %s 2>&1 | FileCheck -check-prefix=RELATIVE_OBJPATH1_COLON %s +// RELATIVE_OBJPATH1_COLON: "-object-file-name=a.obj" + // RUN: %clang_cl -fdebug-compilation-dir=. /Z7 /Fofoo/a.obj -### -- %s 2>&1 | FileCheck -check-prefix=RELATIVE_OBJPATH2 %s // RELATIVE_OBJPATH2: "-object-file-name=foo\\a.obj" -- GitLab From 6634c3e9377abf88c08bb065fb55aa15cda4c248 Mon Sep 17 00:00:00 2001 From: Kai Nacke Date: Mon, 1 Apr 2024 14:20:41 -0400 Subject: [PATCH 0128/1690] [GOFF] Wrap debug output with LLVM_DEBUG (#87252) The content of a GOFF record is always dumped if NDEBUG is not defined, which produces rather confusing output. This changes wrap the dumping code in LLVM_DEBUG, so the dump is only done when debug output of this module is requested. --- llvm/lib/Object/GOFFObjectFile.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Object/GOFFObjectFile.cpp b/llvm/lib/Object/GOFFObjectFile.cpp index 76a13559ebfe..d3dfd5d1540c 100644 --- a/llvm/lib/Object/GOFFObjectFile.cpp +++ b/llvm/lib/Object/GOFFObjectFile.cpp @@ -104,16 +104,13 @@ GOFFObjectFile::GOFFObjectFile(MemoryBufferRef Object, Error &Err) PrevContinuationBits = I[1] & 0x03; continue; } - -#ifndef NDEBUG - for (size_t J = 0; J < GOFF::RecordLength; ++J) { + LLVM_DEBUG(for (size_t J = 0; J < GOFF::RecordLength; ++J) { const uint8_t *P = I + J; if (J % 8 == 0) dbgs() << " "; - dbgs() << format("%02hhX", *P); - } -#endif + }); + switch (RecordType) { case GOFF::RT_ESD: { // Save ESD record. -- GitLab From b8ead2198f27924f91b90b6c104c1234ccc8972e Mon Sep 17 00:00:00 2001 From: Gulfem Savrun Yeniceri Date: Mon, 1 Apr 2024 18:25:02 +0000 Subject: [PATCH 0129/1690] Revert "[CodeGen] Fix register pressure computation in MachinePipeliner (#87030)" This reverts commit a4dec9d6bc67c4d8fbd4a4f54ffaa0399def9627 because the test failed in the following builder: https://luci-milo.appspot.com/ui/p/fuchsia/builders/prod/clang-linux-x64/b8751864477467126481/overview --- llvm/lib/CodeGen/MachinePipeliner.cpp | 2 +- llvm/test/CodeGen/AArch64/sms-regpress.mir | 158 ----------------- llvm/test/CodeGen/PowerPC/sms-regpress.mir | 186 ++++++++++++++++++--- 3 files changed, 166 insertions(+), 180 deletions(-) delete mode 100644 llvm/test/CodeGen/AArch64/sms-regpress.mir diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp index b9c6765be445..eb42a78603d4 100644 --- a/llvm/lib/CodeGen/MachinePipeliner.cpp +++ b/llvm/lib/CodeGen/MachinePipeliner.cpp @@ -1268,7 +1268,7 @@ private: // Calculate the upper limit of each pressure set void computePressureSetLimit(const RegisterClassInfo &RCI) { for (unsigned PSet = 0; PSet < PSetNum; PSet++) - PressureSetLimit[PSet] = TRI->getRegPressureSetLimit(MF, PSet); + PressureSetLimit[PSet] = RCI.getRegPressureSetLimit(PSet); // We assume fixed registers, such as stack pointer, are already in use. // Therefore subtracting the weight of the fixed registers from the limit of diff --git a/llvm/test/CodeGen/AArch64/sms-regpress.mir b/llvm/test/CodeGen/AArch64/sms-regpress.mir deleted file mode 100644 index ad98d5c6124f..000000000000 --- a/llvm/test/CodeGen/AArch64/sms-regpress.mir +++ /dev/null @@ -1,158 +0,0 @@ -# RUN: llc --verify-machineinstrs -mtriple=aarch64 -o - %s -run-pass pipeliner -aarch64-enable-pipeliner -pipeliner-max-mii=40 -pipeliner-register-pressure -pipeliner-ii-search-range=30 -debug-only=pipeliner 2>&1 | FileCheck %s - -# Check that if the register pressure is too high, the schedule is rejected, II is incremented, and scheduling continues. -# The specific value of II is not important. - -# CHECK: {{^ *}}Try to schedule with {{[0-9]+$}} -# CHECK: {{^ *}}Rejected the schedule because of too high register pressure{{$}} -# CHECK: {{^ *}}Try to schedule with {{[0-9]+$}} -# CHECK: {{^ *}}Schedule Found? 1 (II={{[0-9]+}}){{$}} - ---- | - target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" - - define dso_local double @kernel(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %n) local_unnamed_addr { - entry: - %0 = load double, ptr %a, align 8 - %arrayidx1 = getelementptr inbounds i8, ptr %a, i64 8 - %1 = load double, ptr %arrayidx1, align 8 - %cmp133 = icmp sgt i32 %n, 0 - br i1 %cmp133, label %for.body.preheader, label %for.cond.cleanup - - for.body.preheader: ; preds = %entry - %wide.trip.count = zext nneg i32 %n to i64 - br label %for.body - - for.cond.cleanup: ; preds = %for.body, %entry - %res.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add54, %for.body ] - ret double %res.0.lcssa - - for.body: ; preds = %for.body.preheader, %for.body - %lsr.iv137 = phi i64 [ %wide.trip.count, %for.body.preheader ], [ %lsr.iv.next, %for.body ] - %lsr.iv = phi ptr [ %b, %for.body.preheader ], [ %scevgep, %for.body ] - %res.0135 = phi double [ 0.000000e+00, %for.body.preheader ], [ %add54, %for.body ] - %2 = load double, ptr %lsr.iv, align 8 - %3 = tail call double @llvm.fmuladd.f64(double %0, double %2, double %0) - %4 = tail call double @llvm.fmuladd.f64(double %3, double %2, double %3) - %5 = tail call double @llvm.fmuladd.f64(double %4, double %2, double %4) - %6 = tail call double @llvm.fmuladd.f64(double %5, double %2, double %5) - %7 = tail call double @llvm.fmuladd.f64(double %6, double %2, double %6) - %8 = tail call double @llvm.fmuladd.f64(double %7, double %2, double %7) - %9 = tail call double @llvm.fmuladd.f64(double %8, double %2, double %8) - %10 = tail call double @llvm.fmuladd.f64(double %9, double %2, double %9) - %11 = tail call double @llvm.fmuladd.f64(double %10, double %2, double %10) - %12 = tail call double @llvm.fmuladd.f64(double %11, double %2, double %11) - %13 = tail call double @llvm.fmuladd.f64(double %12, double %2, double %12) - %14 = tail call double @llvm.fmuladd.f64(double %13, double %2, double %13) - %15 = tail call double @llvm.fmuladd.f64(double %14, double %2, double %14) - %16 = tail call double @llvm.fmuladd.f64(double %15, double %2, double %15) - %17 = tail call double @llvm.fmuladd.f64(double %16, double %2, double %16) - %18 = tail call double @llvm.fmuladd.f64(double %17, double %2, double %17) - %add = fadd double %17, %18 - %19 = tail call double @llvm.fmuladd.f64(double %18, double %2, double %add) - %add35 = fadd double %10, %19 - %20 = tail call double @llvm.fmuladd.f64(double %3, double %2, double %add35) - %add38 = fadd double %11, %20 - %21 = tail call double @llvm.fmuladd.f64(double %4, double %2, double %add38) - %add41 = fadd double %12, %21 - %22 = tail call double @llvm.fmuladd.f64(double %5, double %2, double %add41) - %add44 = fadd double %14, %15 - %add45 = fadd double %13, %add44 - %add46 = fadd double %add45, %22 - %23 = tail call double @llvm.fmuladd.f64(double %6, double %2, double %add46) - %mul = fmul double %2, %7 - %mul51 = fmul double %1, %mul - %24 = tail call double @llvm.fmuladd.f64(double %mul51, double %9, double %23) - %25 = tail call double @llvm.fmuladd.f64(double %8, double %1, double %24) - %add54 = fadd double %res.0135, %25 - %scevgep = getelementptr i8, ptr %lsr.iv, i64 8 - %lsr.iv.next = add nsw i64 %lsr.iv137, -1 - %exitcond.not = icmp eq i64 %lsr.iv.next, 0 - br i1 %exitcond.not, label %for.cond.cleanup, label %for.body - } - - declare double @llvm.fmuladd.f64(double, double, double) - -... ---- -name: kernel -tracksRegLiveness: true -liveins: - - { reg: '$x0', virtual-reg: '%10' } - - { reg: '$x1', virtual-reg: '%11' } - - { reg: '$w2', virtual-reg: '%12' } -body: | - bb.0.entry: - successors: %bb.1, %bb.4 - liveins: $x0, $x1, $w2 - - %12:gpr32common = COPY $w2 - %11:gpr64 = COPY $x1 - %10:gpr64common = COPY $x0 - dead $wzr = SUBSWri %12, 1, 0, implicit-def $nzcv - Bcc 10, %bb.1, implicit $nzcv - - bb.4: - %13:fpr64 = FMOVD0 - B %bb.2 - - bb.1.for.body.preheader: - %0:fpr64 = LDRDui %10, 0 :: (load (s64) from %ir.a) - %1:fpr64 = LDRDui %10, 1 :: (load (s64) from %ir.arrayidx1) - %16:gpr32 = ORRWrs $wzr, %12, 0 - %2:gpr64all = SUBREG_TO_REG 0, killed %16, %subreg.sub_32 - %15:fpr64 = FMOVD0 - B %bb.3 - - bb.2.for.cond.cleanup: - %3:fpr64 = PHI %13, %bb.4, %7, %bb.3 - $d0 = COPY %3 - RET_ReallyLR implicit $d0 - - bb.3.for.body: - successors: %bb.2, %bb.3 - - %4:gpr64sp = PHI %2, %bb.1, %9, %bb.3 - %5:gpr64sp = PHI %11, %bb.1, %8, %bb.3 - %6:fpr64 = PHI %15, %bb.1, %7, %bb.3 - early-clobber %17:gpr64sp, %18:fpr64 = LDRDpost %5, 8 :: (load (s64) from %ir.lsr.iv) - %19:fpr64 = nofpexcept FMADDDrrr %0, %18, %0, implicit $fpcr - %20:fpr64 = nofpexcept FMADDDrrr %19, %18, %19, implicit $fpcr - %21:fpr64 = nofpexcept FMADDDrrr %20, %18, %20, implicit $fpcr - %22:fpr64 = nofpexcept FMADDDrrr %21, %18, %21, implicit $fpcr - %23:fpr64 = nofpexcept FMADDDrrr %22, %18, %22, implicit $fpcr - %24:fpr64 = nofpexcept FMADDDrrr %23, %18, %23, implicit $fpcr - %25:fpr64 = nofpexcept FMADDDrrr %24, %18, %24, implicit $fpcr - %26:fpr64 = nofpexcept FMADDDrrr %25, %18, %25, implicit $fpcr - %27:fpr64 = nofpexcept FMADDDrrr %26, %18, %26, implicit $fpcr - %28:fpr64 = nofpexcept FMADDDrrr %27, %18, %27, implicit $fpcr - %29:fpr64 = nofpexcept FMADDDrrr %28, %18, %28, implicit $fpcr - %30:fpr64 = nofpexcept FMADDDrrr %29, %18, %29, implicit $fpcr - %31:fpr64 = nofpexcept FMADDDrrr %30, %18, %30, implicit $fpcr - %32:fpr64 = nofpexcept FMADDDrrr %31, %18, %31, implicit $fpcr - %33:fpr64 = nofpexcept FMADDDrrr %32, %18, %32, implicit $fpcr - %34:fpr64 = nofpexcept FMADDDrrr %33, %18, %33, implicit $fpcr - %35:fpr64 = nofpexcept FADDDrr %33, %34, implicit $fpcr - %36:fpr64 = nofpexcept FMADDDrrr %34, %18, killed %35, implicit $fpcr - %37:fpr64 = nofpexcept FADDDrr %26, killed %36, implicit $fpcr - %38:fpr64 = nofpexcept FMADDDrrr %19, %18, killed %37, implicit $fpcr - %39:fpr64 = nofpexcept FADDDrr %27, killed %38, implicit $fpcr - %40:fpr64 = nofpexcept FMADDDrrr %20, %18, killed %39, implicit $fpcr - %41:fpr64 = nofpexcept FADDDrr %28, killed %40, implicit $fpcr - %42:fpr64 = nofpexcept FMADDDrrr %21, %18, killed %41, implicit $fpcr - %43:fpr64 = nofpexcept FADDDrr %30, %31, implicit $fpcr - %44:fpr64 = nofpexcept FADDDrr %29, killed %43, implicit $fpcr - %45:fpr64 = nofpexcept FADDDrr killed %44, killed %42, implicit $fpcr - %46:fpr64 = nofpexcept FMADDDrrr %22, %18, killed %45, implicit $fpcr - %47:fpr64 = nofpexcept FMULDrr %18, %23, implicit $fpcr - %48:fpr64 = nofpexcept FMULDrr %1, killed %47, implicit $fpcr - %49:fpr64 = nofpexcept FMADDDrrr killed %48, %25, killed %46, implicit $fpcr - %50:fpr64 = nofpexcept FMADDDrrr %24, %1, killed %49, implicit $fpcr - %7:fpr64 = nofpexcept FADDDrr %6, killed %50, implicit $fpcr - %8:gpr64all = COPY %17 - %51:gpr64 = nsw SUBSXri %4, 1, 0, implicit-def $nzcv - %9:gpr64all = COPY %51 - Bcc 0, %bb.2, implicit $nzcv - B %bb.3 - -... diff --git a/llvm/test/CodeGen/PowerPC/sms-regpress.mir b/llvm/test/CodeGen/PowerPC/sms-regpress.mir index b01115c49fd8..cebd78af882d 100644 --- a/llvm/test/CodeGen/PowerPC/sms-regpress.mir +++ b/llvm/test/CodeGen/PowerPC/sms-regpress.mir @@ -1,30 +1,41 @@ -# RUN: llc --verify-machineinstrs -mcpu=pwr9 -o - %s -run-pass=pipeliner -ppc-enable-pipeliner -pipeliner-register-pressure -pipeliner-max-mii=50 -pipeliner-ii-search-range=30 -pipeliner-max-stages=10 -debug-only=pipeliner 2>&1 | FileCheck %s +# RUN: llc --verify-machineinstrs -mcpu=pwr9 -o - %s -run-pass=pipeliner -ppc-enable-pipeliner -pipeliner-register-pressure -pipeliner-max-mii=50 -pipeliner-ii-search-range=30 -pipeliner-max-stages=10 -debug-only=pipeliner 2>&1 | FileCheck %s # REQUIRES: asserts # Check that if the register pressure is too high, the schedule is rejected, II is incremented, and scheduling continues. # The specific value of II is not important. -# CHECK: {{^ *}}Try to schedule with {{[0-9]+$}} -# CHECK: {{^ *}}Rejected the schedule because of too high register pressure{{$}} -# CHECK: {{^ *}}Try to schedule with {{[0-9]+$}} -# CHECK: {{^ *}}Schedule Found? 1 (II={{[0-9]+}}){{$}} +# CHECK: Try to schedule with 21 +# CHECK: Can't schedule +# CHECK: Try to schedule with 22 +# CHECK: Can't schedule +# CHECK: Try to schedule with 23 +# CHECK: Rejected the schedule because of too high register pressure +# CHECK: Try to schedule with 24 +# CHECK: Rejected the schedule because of too high register pressure +# CHECK: Try to schedule with 25 +# CHECK: Rejected the schedule because of too high register pressure +# CHECK: Try to schedule with 26 +# CHECK: Schedule Found? 1 (II=26) --- | + ; ModuleID = 'a.ll' + source_filename = "a.c" target datalayout = "e-m:e-Fn32-i64:64-n32:64" target triple = "ppc64le" - define dso_local double @kernel(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef signext %n) local_unnamed_addr { + ; Function Attrs: nofree nosync nounwind memory(argmem: read) uwtable + define dso_local double @kernel(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef signext %n) local_unnamed_addr #0 { entry: - %0 = load double, ptr %a, align 8 - %arrayidx1 = getelementptr inbounds i8, ptr %a, i64 8 - %1 = load double, ptr %arrayidx1, align 8 + %0 = load double, ptr %a, align 8, !tbaa !3 + %arrayidx1 = getelementptr inbounds double, ptr %a, i64 1 + %1 = load double, ptr %arrayidx1, align 8, !tbaa !3 %cmp163 = icmp sgt i32 %n, 0 br i1 %cmp163, label %for.body.preheader, label %for.cond.cleanup for.body.preheader: ; preds = %entry - %wide.trip.count = zext nneg i32 %n to i64 - %scevgep167 = getelementptr i8, ptr %b, i64 -8 + %wide.trip.count = zext i32 %n to i64 + %scevgep1 = getelementptr i8, ptr %b, i64 -8 call void @llvm.set.loop.iterations.i64(i64 %wide.trip.count) br label %for.body @@ -32,11 +43,11 @@ %res.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %30, %for.body ] ret double %res.0.lcssa - for.body: ; preds = %for.body.preheader, %for.body + for.body: ; preds = %for.body, %for.body.preheader %res.0165 = phi double [ 0.000000e+00, %for.body.preheader ], [ %30, %for.body ] - %2 = phi ptr [ %scevgep167, %for.body.preheader ], [ %3, %for.body ] + %2 = phi ptr [ %scevgep1, %for.body.preheader ], [ %3, %for.body ] %3 = getelementptr i8, ptr %2, i64 8 - %4 = load double, ptr %3, align 8 + %4 = load double, ptr %3, align 8, !tbaa !3 %5 = tail call double @llvm.fmuladd.f64(double %0, double %4, double %0) %6 = tail call double @llvm.fmuladd.f64(double %5, double %4, double %5) %7 = tail call double @llvm.fmuladd.f64(double %6, double %4, double %6) @@ -81,23 +92,152 @@ %mul66 = fmul double %12, %mul65 %30 = tail call double @llvm.fmuladd.f64(double %mul66, double %10, double %res.0165) %31 = call i1 @llvm.loop.decrement.i64(i64 1) - br i1 %31, label %for.body, label %for.cond.cleanup + br i1 %31, label %for.body, label %for.cond.cleanup, !llvm.loop !7 } - declare double @llvm.fmuladd.f64(double, double, double) + ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) + declare double @llvm.fmuladd.f64(double, double, double) #1 - declare void @llvm.set.loop.iterations.i64(i64) + ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn + declare void @llvm.set.loop.iterations.i64(i64) #2 - declare i1 @llvm.loop.decrement.i64(i64) + ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn + declare i1 @llvm.loop.decrement.i64(i64) #2 + attributes #0 = { nofree nosync nounwind memory(argmem: read) uwtable "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crbits,+crypto,+direct-move,+extdiv,+htm,+isa-v206-instructions,+isa-v207-instructions,+isa-v30-instructions,+power8-vector,+power9-vector,+quadword-atomics,+vsx,-aix-small-local-exec-tls,-privileged,-rop-protect,-spe" } + attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + attributes #2 = { nocallback noduplicate nofree nosync nounwind willreturn } + + !llvm.module.flags = !{!0, !1} + !llvm.ident = !{!2} + + !0 = !{i32 1, !"wchar_size", i32 4} + !1 = !{i32 7, !"uwtable", i32 2} + !2 = !{!"clang version 18.0.0 (https://miratech-soft@dev.azure.com/miratech-soft/llvm/_git/llvm c8d01fb665fc5d9378100a6d92ebcd3be49be655)"} + !3 = !{!4, !4, i64 0} + !4 = !{!"double", !5, i64 0} + !5 = !{!"omnipotent char", !6, i64 0} + !6 = !{!"Simple C/C++ TBAA"} + !7 = distinct !{!7, !8, !9} + !8 = !{!"llvm.loop.mustprogress"} + !9 = !{!"llvm.loop.unroll.disable"} + ... --- name: kernel +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +failedISel: false tracksRegLiveness: true +hasWinCFI: false +callsEHReturn: false +callsUnwindInit: false +hasEHCatchret: false +hasEHScopes: false +hasEHFunclets: false +isOutlined: false +debugInstrRef: false +failsVerification: false +tracksDebugUserValues: false +registers: + - { id: 0, class: vsfrc, preferred-register: '' } + - { id: 1, class: vsfrc, preferred-register: '' } + - { id: 2, class: g8rc, preferred-register: '' } + - { id: 3, class: vsfrc, preferred-register: '' } + - { id: 4, class: vsfrc, preferred-register: '' } + - { id: 5, class: g8rc_and_g8rc_nox0, preferred-register: '' } + - { id: 6, class: g8rc, preferred-register: '' } + - { id: 7, class: vsfrc, preferred-register: '' } + - { id: 8, class: g8rc_and_g8rc_nox0, preferred-register: '' } + - { id: 9, class: g8rc_and_g8rc_nox0, preferred-register: '' } + - { id: 10, class: g8rc, preferred-register: '' } + - { id: 11, class: gprc, preferred-register: '' } + - { id: 12, class: vsfrc, preferred-register: '' } + - { id: 13, class: crrc, preferred-register: '' } + - { id: 14, class: vsfrc, preferred-register: '' } + - { id: 15, class: g8rc, preferred-register: '' } + - { id: 16, class: g8rc, preferred-register: '' } + - { id: 17, class: g8rc, preferred-register: '' } + - { id: 18, class: f8rc, preferred-register: '' } + - { id: 19, class: g8rc_and_g8rc_nox0, preferred-register: '' } + - { id: 20, class: vsfrc, preferred-register: '' } + - { id: 21, class: vsfrc, preferred-register: '' } + - { id: 22, class: vsfrc, preferred-register: '' } + - { id: 23, class: vsfrc, preferred-register: '' } + - { id: 24, class: vsfrc, preferred-register: '' } + - { id: 25, class: vsfrc, preferred-register: '' } + - { id: 26, class: vsfrc, preferred-register: '' } + - { id: 27, class: vsfrc, preferred-register: '' } + - { id: 28, class: vsfrc, preferred-register: '' } + - { id: 29, class: vsfrc, preferred-register: '' } + - { id: 30, class: vsfrc, preferred-register: '' } + - { id: 31, class: vsfrc, preferred-register: '' } + - { id: 32, class: vsfrc, preferred-register: '' } + - { id: 33, class: vsfrc, preferred-register: '' } + - { id: 34, class: vsfrc, preferred-register: '' } + - { id: 35, class: vsfrc, preferred-register: '' } + - { id: 36, class: vsfrc, preferred-register: '' } + - { id: 37, class: vsfrc, preferred-register: '' } + - { id: 38, class: vsfrc, preferred-register: '' } + - { id: 39, class: vsfrc, preferred-register: '' } + - { id: 40, class: vsfrc, preferred-register: '' } + - { id: 41, class: vsfrc, preferred-register: '' } + - { id: 42, class: vsfrc, preferred-register: '' } + - { id: 43, class: vsfrc, preferred-register: '' } + - { id: 44, class: vsfrc, preferred-register: '' } + - { id: 45, class: vsfrc, preferred-register: '' } + - { id: 46, class: vsfrc, preferred-register: '' } + - { id: 47, class: vsfrc, preferred-register: '' } + - { id: 48, class: vsfrc, preferred-register: '' } + - { id: 49, class: vsfrc, preferred-register: '' } + - { id: 50, class: vsfrc, preferred-register: '' } + - { id: 51, class: vsfrc, preferred-register: '' } + - { id: 52, class: vsfrc, preferred-register: '' } + - { id: 53, class: vsfrc, preferred-register: '' } + - { id: 54, class: vsfrc, preferred-register: '' } + - { id: 55, class: vsfrc, preferred-register: '' } + - { id: 56, class: vsfrc, preferred-register: '' } + - { id: 57, class: vsfrc, preferred-register: '' } + - { id: 58, class: vsfrc, preferred-register: '' } + - { id: 59, class: vsfrc, preferred-register: '' } + - { id: 60, class: vsfrc, preferred-register: '' } + - { id: 61, class: vsfrc, preferred-register: '' } + - { id: 62, class: crbitrc, preferred-register: '' } liveins: - { reg: '$x3', virtual-reg: '%8' } - { reg: '$x4', virtual-reg: '%9' } - { reg: '$x5', virtual-reg: '%10' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 1 + adjustsStack: false + hasCalls: false + stackProtector: '' + functionContext: '' + maxCallFrameSize: 4294967295 + cvBytesOfCalleeSavedRegisters: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + hasTailCall: false + localFrameSize: 0 + savePoint: '' + restorePoint: '' +fixedStack: [] +stack: [] +entry_values: [] +callSites: [] +debugValueSubstitutions: [] +constants: [] +machineFunctionInfo: {} body: | bb.0.entry: successors: %bb.2(0x50000000), %bb.1(0x30000000) @@ -111,12 +251,16 @@ body: | BCC 44, killed %13, %bb.2 bb.1: + successors: %bb.3(0x80000000) + %12:vsfrc = XXLXORdpz B %bb.3 bb.2.for.body.preheader: - %0:vsfrc = DFLOADf64 0, %8 :: (load (s64) from %ir.a) - %1:vsfrc = DFLOADf64 8, killed %8 :: (load (s64) from %ir.arrayidx1) + successors: %bb.4(0x80000000) + + %0:vsfrc = DFLOADf64 0, %8 :: (load (s64) from %ir.a, !tbaa !3) + %1:vsfrc = DFLOADf64 8, killed %8 :: (load (s64) from %ir.arrayidx1, !tbaa !3) %16:g8rc = IMPLICIT_DEF %15:g8rc = INSERT_SUBREG killed %16, killed %11, %subreg.sub_32 %17:g8rc = RLDICL killed %15, 0, 32 @@ -135,7 +279,7 @@ body: | %4:vsfrc = PHI %14, %bb.2, %7, %bb.4 %5:g8rc_and_g8rc_nox0 = PHI %2, %bb.2, %6, %bb.4 - %18:f8rc, %19:g8rc_and_g8rc_nox0 = LFDU 8, killed %5 :: (load (s64) from %ir.3) + %18:f8rc, %19:g8rc_and_g8rc_nox0 = LFDU 8, killed %5 :: (load (s64) from %ir.3, !tbaa !3) %6:g8rc = COPY killed %19 %20:vsfrc = nofpexcept XSMADDADP %0, %0, %18, implicit $rm %21:vsfrc = nofpexcept XSMADDADP %20, %20, %18, implicit $rm -- GitLab From e45f6e569dafd4033f86d276065d77799b5f6226 Mon Sep 17 00:00:00 2001 From: Vlad Serebrennikov Date: Mon, 1 Apr 2024 22:37:37 +0400 Subject: [PATCH 0130/1690] [clang] Factor out OpenACC part of `Sema` (#84184) This patch moves OpenACC parts of `Sema` into a separate class `SemaOpenACC` that is placed in a separate header `Sema/SemaOpenACC.h`. This patch is intended to be a model of factoring things out of `Sema`, so I picked a small OpenACC part. Goals are the following: 1) Split `Sema` into manageable parts. 2) Make dependencies between parts visible. 3) Improve Clang development cycle by avoiding recompiling unrelated parts of the compiler. 4) Avoid compile-time regressions. 5) Avoid notational regressions in the code that uses Sema. --- clang/include/clang/Sema/Sema.h | 65 ++++------------------ clang/include/clang/Sema/SemaOpenACC.h | 74 ++++++++++++++++++++++++++ clang/lib/Parse/ParseOpenACC.cpp | 21 ++++---- clang/lib/Sema/JumpDiagnostics.cpp | 1 + clang/lib/Sema/Sema.cpp | 3 +- clang/lib/Sema/SemaOpenACC.cpp | 33 +++++++----- clang/lib/Sema/TreeTransform.h | 11 ++-- 7 files changed, 125 insertions(+), 83 deletions(-) create mode 100644 clang/include/clang/Sema/SemaOpenACC.h diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 3a1abd4c7892..a02b684f2c77 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -33,7 +33,6 @@ #include "clang/AST/NSAPI.h" #include "clang/AST/PrettyPrinter.h" #include "clang/AST/StmtCXX.h" -#include "clang/AST/StmtOpenACC.h" #include "clang/AST/StmtOpenMP.h" #include "clang/AST/TypeLoc.h" #include "clang/AST/TypeOrdering.h" @@ -42,7 +41,6 @@ #include "clang/Basic/DarwinSDKInfo.h" #include "clang/Basic/ExpressionTraits.h" #include "clang/Basic/Module.h" -#include "clang/Basic/OpenACCKinds.h" #include "clang/Basic/OpenCLOptions.h" #include "clang/Basic/OpenMPKinds.h" #include "clang/Basic/PragmaKinds.h" @@ -183,6 +181,7 @@ class Preprocessor; class PseudoDestructorTypeStorage; class PseudoObjectExpr; class QualType; +class SemaOpenACC; class StandardConversionSequence; class Stmt; class StringLiteral; @@ -466,9 +465,8 @@ class Sema final { // 37. Name Lookup for RISC-V Vector Intrinsic (SemaRISCVVectorLookup.cpp) // 38. CUDA (SemaCUDA.cpp) // 39. HLSL Constructs (SemaHLSL.cpp) - // 40. OpenACC Constructs (SemaOpenACC.cpp) - // 41. OpenMP Directives and Clauses (SemaOpenMP.cpp) - // 42. SYCL Constructs (SemaSYCL.cpp) + // 40. OpenMP Directives and Clauses (SemaOpenMP.cpp) + // 41. SYCL Constructs (SemaSYCL.cpp) /// \name Semantic Analysis /// Implementations are in Sema.cpp @@ -1162,6 +1160,11 @@ public: /// CurContext - This is the current declaration context of parsing. DeclContext *CurContext; + SemaOpenACC &OpenACC() { + assert(OpenACCPtr); + return *OpenACCPtr; + } + protected: friend class Parser; friend class InitializationSequence; @@ -1192,6 +1195,8 @@ private: mutable IdentifierInfo *Ident_super; + std::unique_ptr OpenACCPtr; + ///@} // @@ -13351,56 +13356,6 @@ public: // // - /// \name OpenACC Constructs - /// Implementations are in SemaOpenACC.cpp - ///@{ - -public: - /// Called after parsing an OpenACC Clause so that it can be checked. - bool ActOnOpenACCClause(OpenACCClauseKind ClauseKind, - SourceLocation StartLoc); - - /// Called after the construct has been parsed, but clauses haven't been - /// parsed. This allows us to diagnose not-implemented, as well as set up any - /// state required for parsing the clauses. - void ActOnOpenACCConstruct(OpenACCDirectiveKind K, SourceLocation StartLoc); - - /// Called after the directive, including its clauses, have been parsed and - /// parsing has consumed the 'annot_pragma_openacc_end' token. This DOES - /// happen before any associated declarations or statements have been parsed. - /// This function is only called when we are parsing a 'statement' context. - bool ActOnStartOpenACCStmtDirective(OpenACCDirectiveKind K, - SourceLocation StartLoc); - - /// Called after the directive, including its clauses, have been parsed and - /// parsing has consumed the 'annot_pragma_openacc_end' token. This DOES - /// happen before any associated declarations or statements have been parsed. - /// This function is only called when we are parsing a 'Decl' context. - bool ActOnStartOpenACCDeclDirective(OpenACCDirectiveKind K, - SourceLocation StartLoc); - /// Called when we encounter an associated statement for our construct, this - /// should check legality of the statement as it appertains to this Construct. - StmtResult ActOnOpenACCAssociatedStmt(OpenACCDirectiveKind K, - StmtResult AssocStmt); - - /// Called after the directive has been completely parsed, including the - /// declaration group or associated statement. - StmtResult ActOnEndOpenACCStmtDirective(OpenACCDirectiveKind K, - SourceLocation StartLoc, - SourceLocation EndLoc, - StmtResult AssocStmt); - /// Called after the directive has been completely parsed, including the - /// declaration group or associated statement. - DeclGroupRef ActOnEndOpenACCDeclDirective(); - - ///@} - - // - // - // ------------------------------------------------------------------------- - // - // - /// \name OpenMP Directives and Clauses /// Implementations are in SemaOpenMP.cpp ///@{ diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h new file mode 100644 index 000000000000..7f50d7889ad7 --- /dev/null +++ b/clang/include/clang/Sema/SemaOpenACC.h @@ -0,0 +1,74 @@ +//===----- SemaOpenACC.h - Semantic Analysis for OpenACC constructs -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file declares semantic analysis for OpenACC constructs and +/// clauses. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_SEMA_SEMAOPENACC_H +#define LLVM_CLANG_SEMA_SEMAOPENACC_H + +#include "clang/AST/DeclGroup.h" +#include "clang/Basic/OpenACCKinds.h" +#include "clang/Basic/SourceLocation.h" +#include "clang/Sema/Ownership.h" + +namespace clang { + +class ASTContext; +class DiagnosticEngine; +class LangOptions; +class Sema; + +class SemaOpenACC { +public: + SemaOpenACC(Sema &S); + + ASTContext &getASTContext() const; + DiagnosticsEngine &getDiagnostics() const; + const LangOptions &getLangOpts() const; + + Sema &SemaRef; + + /// Called after parsing an OpenACC Clause so that it can be checked. + bool ActOnClause(OpenACCClauseKind ClauseKind, SourceLocation StartLoc); + + /// Called after the construct has been parsed, but clauses haven't been + /// parsed. This allows us to diagnose not-implemented, as well as set up any + /// state required for parsing the clauses. + void ActOnConstruct(OpenACCDirectiveKind K, SourceLocation StartLoc); + + /// Called after the directive, including its clauses, have been parsed and + /// parsing has consumed the 'annot_pragma_openacc_end' token. This DOES + /// happen before any associated declarations or statements have been parsed. + /// This function is only called when we are parsing a 'statement' context. + bool ActOnStartStmtDirective(OpenACCDirectiveKind K, SourceLocation StartLoc); + + /// Called after the directive, including its clauses, have been parsed and + /// parsing has consumed the 'annot_pragma_openacc_end' token. This DOES + /// happen before any associated declarations or statements have been parsed. + /// This function is only called when we are parsing a 'Decl' context. + bool ActOnStartDeclDirective(OpenACCDirectiveKind K, SourceLocation StartLoc); + /// Called when we encounter an associated statement for our construct, this + /// should check legality of the statement as it appertains to this Construct. + StmtResult ActOnAssociatedStmt(OpenACCDirectiveKind K, StmtResult AssocStmt); + + /// Called after the directive has been completely parsed, including the + /// declaration group or associated statement. + StmtResult ActOnEndStmtDirective(OpenACCDirectiveKind K, + SourceLocation StartLoc, + SourceLocation EndLoc, StmtResult AssocStmt); + /// Called after the directive has been completely parsed, including the + /// declaration group or associated statement. + DeclGroupRef ActOnEndDeclDirective(); +}; + +} // namespace clang + +#endif // LLVM_CLANG_SEMA_SEMAOPENACC_H diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp index 50e3c39f6091..07dd2ba0106a 100644 --- a/clang/lib/Parse/ParseOpenACC.cpp +++ b/clang/lib/Parse/ParseOpenACC.cpp @@ -14,6 +14,7 @@ #include "clang/Parse/ParseDiagnostic.h" #include "clang/Parse/Parser.h" #include "clang/Parse/RAIIObjectsForParser.h" +#include "clang/Sema/SemaOpenACC.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSwitch.h" @@ -777,7 +778,7 @@ bool Parser::ParseOpenACCClause(OpenACCDirectiveKind DirKind) { SourceLocation ClauseLoc = ConsumeToken(); bool Result = ParseOpenACCClauseParams(DirKind, Kind); - getActions().ActOnOpenACCClause(Kind, ClauseLoc); + getActions().OpenACC().ActOnClause(Kind, ClauseLoc); return Result; } @@ -1151,7 +1152,7 @@ Parser::OpenACCDirectiveParseInfo Parser::ParseOpenACCDirective() { SourceLocation StartLoc = getCurToken().getLocation(); OpenACCDirectiveKind DirKind = ParseOpenACCDirectiveKind(*this); - getActions().ActOnOpenACCConstruct(DirKind, StartLoc); + getActions().OpenACC().ActOnConstruct(DirKind, StartLoc); // Once we've parsed the construct/directive name, some have additional // specifiers that need to be taken care of. Atomic has an 'atomic-clause' @@ -1223,12 +1224,12 @@ Parser::DeclGroupPtrTy Parser::ParseOpenACCDirectiveDecl() { OpenACCDirectiveParseInfo DirInfo = ParseOpenACCDirective(); - if (getActions().ActOnStartOpenACCDeclDirective(DirInfo.DirKind, - DirInfo.StartLoc)) + if (getActions().OpenACC().ActOnStartDeclDirective(DirInfo.DirKind, + DirInfo.StartLoc)) return nullptr; // TODO OpenACC: Do whatever decl parsing is required here. - return DeclGroupPtrTy::make(getActions().ActOnEndOpenACCDeclDirective()); + return DeclGroupPtrTy::make(getActions().OpenACC().ActOnEndDeclDirective()); } // Parse OpenACC Directive on a Statement. @@ -1239,8 +1240,8 @@ StmtResult Parser::ParseOpenACCDirectiveStmt() { ConsumeAnnotationToken(); OpenACCDirectiveParseInfo DirInfo = ParseOpenACCDirective(); - if (getActions().ActOnStartOpenACCStmtDirective(DirInfo.DirKind, - DirInfo.StartLoc)) + if (getActions().OpenACC().ActOnStartStmtDirective(DirInfo.DirKind, + DirInfo.StartLoc)) return StmtError(); StmtResult AssocStmt; @@ -1249,10 +1250,10 @@ StmtResult Parser::ParseOpenACCDirectiveStmt() { ParsingOpenACCDirectiveRAII DirScope(*this, /*Value=*/false); ParseScope ACCScope(this, getOpenACCScopeFlags(DirInfo.DirKind)); - AssocStmt = getActions().ActOnOpenACCAssociatedStmt(DirInfo.DirKind, - ParseStatement()); + AssocStmt = getActions().OpenACC().ActOnAssociatedStmt(DirInfo.DirKind, + ParseStatement()); } - return getActions().ActOnEndOpenACCStmtDirective( + return getActions().OpenACC().ActOnEndStmtDirective( DirInfo.DirKind, DirInfo.StartLoc, DirInfo.EndLoc, AssocStmt); } diff --git a/clang/lib/Sema/JumpDiagnostics.cpp b/clang/lib/Sema/JumpDiagnostics.cpp index 6722878883be..ce6211c23218 100644 --- a/clang/lib/Sema/JumpDiagnostics.cpp +++ b/clang/lib/Sema/JumpDiagnostics.cpp @@ -16,6 +16,7 @@ #include "clang/AST/ExprCXX.h" #include "clang/AST/StmtCXX.h" #include "clang/AST/StmtObjC.h" +#include "clang/AST/StmtOpenACC.h" #include "clang/AST/StmtOpenMP.h" #include "clang/Basic/SourceLocation.h" #include "clang/Sema/SemaInternal.h" diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp index c9dbac0dfc33..b7e4fc0ac9b5 100644 --- a/clang/lib/Sema/Sema.cpp +++ b/clang/lib/Sema/Sema.cpp @@ -43,6 +43,7 @@ #include "clang/Sema/ScopeInfo.h" #include "clang/Sema/SemaConsumer.h" #include "clang/Sema/SemaInternal.h" +#include "clang/Sema/SemaOpenACC.h" #include "clang/Sema/TemplateDeduction.h" #include "clang/Sema/TemplateInstCallback.h" #include "clang/Sema/TypoCorrection.h" @@ -196,7 +197,7 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer, ThreadSafetyDeclCache(nullptr), LateTemplateParser(nullptr), LateTemplateParserCleanup(nullptr), OpaqueParser(nullptr), CurContext(nullptr), ExternalSource(nullptr), CurScope(nullptr), - Ident_super(nullptr), + Ident_super(nullptr), OpenACCPtr(std::make_unique(*this)), MSPointerToMemberRepresentationMethod( LangOpts.getMSPointerToMemberRepresentationMethod()), MSStructPragmaOn(false), VtorDispStack(LangOpts.getVtorDispMode()), diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp index d3a602d1c382..2ac994cac71e 100644 --- a/clang/lib/Sema/SemaOpenACC.cpp +++ b/clang/lib/Sema/SemaOpenACC.cpp @@ -11,14 +11,15 @@ /// //===----------------------------------------------------------------------===// +#include "clang/AST/StmtOpenACC.h" +#include "clang/Sema/SemaOpenACC.h" #include "clang/Basic/DiagnosticSema.h" -#include "clang/Basic/OpenACCKinds.h" #include "clang/Sema/Sema.h" using namespace clang; namespace { -bool diagnoseConstructAppertainment(Sema &S, OpenACCDirectiveKind K, +bool diagnoseConstructAppertainment(SemaOpenACC &S, OpenACCDirectiveKind K, SourceLocation StartLoc, bool IsStmt) { switch (K) { default: @@ -30,14 +31,21 @@ bool diagnoseConstructAppertainment(Sema &S, OpenACCDirectiveKind K, case OpenACCDirectiveKind::Serial: case OpenACCDirectiveKind::Kernels: if (!IsStmt) - return S.Diag(StartLoc, diag::err_acc_construct_appertainment) << K; + return S.SemaRef.Diag(StartLoc, diag::err_acc_construct_appertainment) + << K; break; } return false; } } // namespace -bool Sema::ActOnOpenACCClause(OpenACCClauseKind ClauseKind, +SemaOpenACC::SemaOpenACC(Sema &S) : SemaRef(S) {} + +ASTContext &SemaOpenACC::getASTContext() const { return SemaRef.Context; } +DiagnosticsEngine &SemaOpenACC::getDiagnostics() const { return SemaRef.Diags; } +const LangOptions &SemaOpenACC::getLangOpts() const { return SemaRef.LangOpts; } + +bool SemaOpenACC::ActOnClause(OpenACCClauseKind ClauseKind, SourceLocation StartLoc) { if (ClauseKind == OpenACCClauseKind::Invalid) return false; @@ -45,9 +53,10 @@ bool Sema::ActOnOpenACCClause(OpenACCClauseKind ClauseKind, // whatever it can do. This function will eventually need to start returning // some sort of Clause AST type, but for now just return true/false based on // success. - return Diag(StartLoc, diag::warn_acc_clause_unimplemented) << ClauseKind; + return SemaRef.Diag(StartLoc, diag::warn_acc_clause_unimplemented) + << ClauseKind; } -void Sema::ActOnOpenACCConstruct(OpenACCDirectiveKind K, +void SemaOpenACC::ActOnConstruct(OpenACCDirectiveKind K, SourceLocation StartLoc) { switch (K) { case OpenACCDirectiveKind::Invalid: @@ -63,17 +72,17 @@ void Sema::ActOnOpenACCConstruct(OpenACCDirectiveKind K, // here as these constructs do not take any arguments. break; default: - Diag(StartLoc, diag::warn_acc_construct_unimplemented) << K; + SemaRef.Diag(StartLoc, diag::warn_acc_construct_unimplemented) << K; break; } } -bool Sema::ActOnStartOpenACCStmtDirective(OpenACCDirectiveKind K, +bool SemaOpenACC::ActOnStartStmtDirective(OpenACCDirectiveKind K, SourceLocation StartLoc) { return diagnoseConstructAppertainment(*this, K, StartLoc, /*IsStmt=*/true); } -StmtResult Sema::ActOnEndOpenACCStmtDirective(OpenACCDirectiveKind K, +StmtResult SemaOpenACC::ActOnEndStmtDirective(OpenACCDirectiveKind K, SourceLocation StartLoc, SourceLocation EndLoc, StmtResult AssocStmt) { @@ -92,7 +101,7 @@ StmtResult Sema::ActOnEndOpenACCStmtDirective(OpenACCDirectiveKind K, llvm_unreachable("Unhandled case in directive handling?"); } -StmtResult Sema::ActOnOpenACCAssociatedStmt(OpenACCDirectiveKind K, +StmtResult SemaOpenACC::ActOnAssociatedStmt(OpenACCDirectiveKind K, StmtResult AssocStmt) { switch (K) { default: @@ -114,9 +123,9 @@ StmtResult Sema::ActOnOpenACCAssociatedStmt(OpenACCDirectiveKind K, llvm_unreachable("Invalid associated statement application"); } -bool Sema::ActOnStartOpenACCDeclDirective(OpenACCDirectiveKind K, +bool SemaOpenACC::ActOnStartDeclDirective(OpenACCDirectiveKind K, SourceLocation StartLoc) { return diagnoseConstructAppertainment(*this, K, StartLoc, /*IsStmt=*/false); } -DeclGroupRef Sema::ActOnEndOpenACCDeclDirective() { return DeclGroupRef{}; } +DeclGroupRef SemaOpenACC::ActOnEndDeclDirective() { return DeclGroupRef{}; } diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h index eace1bfdff5a..a2568ad0f82c 100644 --- a/clang/lib/Sema/TreeTransform.h +++ b/clang/lib/Sema/TreeTransform.h @@ -39,6 +39,7 @@ #include "clang/Sema/ScopeInfo.h" #include "clang/Sema/SemaDiagnostic.h" #include "clang/Sema/SemaInternal.h" +#include "clang/Sema/SemaOpenACC.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/Support/ErrorHandling.h" #include @@ -4000,16 +4001,16 @@ public: SourceLocation BeginLoc, SourceLocation EndLoc, StmtResult StrBlock) { - getSema().ActOnOpenACCConstruct(K, BeginLoc); + getSema().OpenACC().ActOnConstruct(K, BeginLoc); // TODO OpenACC: Include clauses. - if (getSema().ActOnStartOpenACCStmtDirective(K, BeginLoc)) + if (getSema().OpenACC().ActOnStartStmtDirective(K, BeginLoc)) return StmtError(); - StrBlock = getSema().ActOnOpenACCAssociatedStmt(K, StrBlock); + StrBlock = getSema().OpenACC().ActOnAssociatedStmt(K, StrBlock); - return getSema().ActOnEndOpenACCStmtDirective(K, BeginLoc, EndLoc, - StrBlock); + return getSema().OpenACC().ActOnEndStmtDirective(K, BeginLoc, EndLoc, + StrBlock); } private: -- GitLab From f3ec73fca492124b15c3eb9a3ae12b7d86470d27 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 1 Apr 2024 11:47:11 -0700 Subject: [PATCH 0131/1690] [NFC]Precommit test for vtable import (#79363) A precommit test case to show function summary and global values when a function has instructions annotated with vtable profiles and indirect call profiles. - This is a precommit test for https://github.com/llvm/llvm-project/pull/79381 --- .../thinlto-func-summary-vtableref-pgo.ll | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll diff --git a/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll new file mode 100644 index 000000000000..78b175caca85 --- /dev/null +++ b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll @@ -0,0 +1,61 @@ +; RUN: opt -module-summary %s -o %t.o + +; RUN: llvm-bcanalyzer -dump %t.o | FileCheck %s + +; RUN: llvm-dis -o - %t.o | FileCheck %s --check-prefix=DIS + + +; CHECK: +; CHECK-NEXT: +; The `VALUE_GUID` below represents the "_ZN4Base4funcEv" referenced by the +; indirect call instruction. +; CHECK-NEXT: +; has the format [valueid, flags, instcount, funcflags, +; numrefs, rorefcnt, worefcnt, +; n x (valueid, hotness+tailcall)] +; CHECK-NEXT: +; CHECK-NEXT: + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function has one BB and an entry count of 150, so the BB is hot according to +; ProfileSummary and reflected so in the bitcode (see llvm-dis output). +define i32 @_Z4testP4Base(ptr %0) !prof !15 { + %2 = load ptr, ptr %0, !prof !16 + %3 = load ptr, ptr %2 + %4 = tail call i32 %3(ptr %0), !prof !17 + ret i32 %4 +} + +!llvm.module.flags = !{!1} + + +!1 = !{i32 1, !"ProfileSummary", !2} +!2 = !{!3, !4, !5, !6, !7, !8, !9, !10} +!3 = !{!"ProfileFormat", !"InstrProf"} +!4 = !{!"TotalCount", i64 10000} +!5 = !{!"MaxCount", i64 200} +!6 = !{!"MaxInternalCount", i64 200} +!7 = !{!"MaxFunctionCount", i64 200} +!8 = !{!"NumCounts", i64 3} +!9 = !{!"NumFunctions", i64 3} +!10 = !{!"DetailedSummary", !11} +!11 = !{!12, !13, !14} +!12 = !{i32 10000, i64 100, i32 1} +!13 = !{i32 990000, i64 100, i32 1} +!14 = !{i32 999999, i64 1, i32 2} + +!15 = !{!"function_entry_count", i32 150} +; 1960855528937986108 is the MD5 hash of _ZTV4Base +!16 = !{!"VP", i32 2, i64 1600, i64 1960855528937986108, i64 1600} +; 5459407273543877811 is the MD5 hash of _ZN4Base4funcEv +!17 = !{!"VP", i32 0, i64 1600, i64 5459407273543877811, i64 1600} + +; ModuleSummaryIndex stores map in std::map; so +; global value summares are printed out in the order that gv's guid increases. +; DIS: ^0 = module: (path: "{{.*}}", hash: (0, 0, 0, 0, 0)) +; DIS: ^1 = gv: (guid: 5459407273543877811) +; DIS: ^2 = gv: (name: "_Z4testP4Base", summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 4, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 1, mustBeUnreachable: 0), calls: ((callee: ^1, hotness: hot))))) ; guid = 15857150948103218965 +; DIS: ^3 = blockcount: 0 -- GitLab From cbd48b184eca1ca73e6f20575501d94ad30fbd58 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Mon, 1 Apr 2024 14:49:56 -0400 Subject: [PATCH 0132/1690] [C99] Claim conformance to "conversion of array to pointer not limited to lvalues" We don't have a document number for this, but the change was called out explicitly in the editor's comments in the C99 foreword. --- clang/test/C/C99/array-lvalue.c | 38 +++++++++++++++++++++++++++++++++ clang/www/c_status.html | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 clang/test/C/C99/array-lvalue.c diff --git a/clang/test/C/C99/array-lvalue.c b/clang/test/C/C99/array-lvalue.c new file mode 100644 index 000000000000..4e963b4f74fc --- /dev/null +++ b/clang/test/C/C99/array-lvalue.c @@ -0,0 +1,38 @@ +/* RUN: %clang_cc1 -verify -pedantic -std=c99 %s + RUN: %clang_cc1 -verify=c89 -pedantic -std=c89 %s + expected-no-diagnostics + */ + +/* WG14 ???: Yes + * Conversion of array to pointer not limited to lvalues + * + * NB: the original paper number is unknown, this was gleaned from the editor's report + * in the C99 foreword. The C99 rationale document did not shed much light on + * the situation either, mostly talking about user confusion between lvalue and + * modifiable lvalue. However, the crux of the change was C89 changing: + * + * C89 3.2.2.1: Except when it is the operand of ..., an lvalue that has type + * 'array of type' is converted to an expression that has type 'pointer to + * type' that points to the initial element of the array object and is not an + * lvalue. + * + * C99 6.3.2.1p3: Except when it is the operand of ..., an expression that has + * type 'array of type' is converted to an expression with type 'pointer to + * type' that points to the initial element of the array object and is not an + * lvalue. + */ + +struct S { + char arr[100]; +}; + +struct S f(void); + +void func(void) { + char c; + /* The return from f() is an rvalue, so this code is not valid in C89, but is + * valid in C99. + */ + c = f().arr[10]; /* c89-warning {{ISO C90 does not allow subscripting non-lvalue array}} */ +} + diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 123897593e5d..803dce8e29fc 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -360,7 +360,7 @@ conformance.

conversion of array to pointer not limited to lvalues Unknown - Unknown + Yes relaxed constraints on aggregate and union initialization -- GitLab From 4cd7bb07c7540bf83a7a60a67aa282e99461ca2f Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Mon, 1 Apr 2024 22:07:10 +0300 Subject: [PATCH 0133/1690] [mlir] Remove ``dataclasses`` package from mlir ``requirements.txt`` (#87223) The ``dataclasses`` package makes sense for Python 3.6, becauses ``dataclasses`` is only included in the standard library with 3.7 version. Now, 3.6 has reached EOL, so all current supported versions of Python (3.8, 3.9, 3.10, 3.11, 3.12) have this feature in their standard libraries. Therefore there's no need to install the ``dataclasses`` package now. --- mlir/python/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mlir/python/requirements.txt b/mlir/python/requirements.txt index a596f8747ebe..acd6dbb25eda 100644 --- a/mlir/python/requirements.txt +++ b/mlir/python/requirements.txt @@ -1,4 +1,3 @@ numpy>=1.19.5, <=1.26 pybind11>=2.9.0, <=2.10.3 -PyYAML>=5.3.1, <=6.0.1 -dataclasses>=0.6, <=0.8 \ No newline at end of file +PyYAML>=5.3.1, <=6.0.1 \ No newline at end of file -- GitLab From ee99475068523de185dce0a449b65e684a1e6b73 Mon Sep 17 00:00:00 2001 From: Nathan Sidwell Date: Mon, 1 Apr 2024 15:41:38 -0400 Subject: [PATCH 0134/1690] [clang] Fix bitfield access unit for vbase corner case (#87238) This fixes #87227, a vbase can be placed below nvsize when empty members and/or bases are in play. We must account for that. --- clang/lib/CodeGen/CGRecordLayoutBuilder.cpp | 57 +++++++--- .../test/CodeGenCXX/bitfield-access-tail.cpp | 104 ++++++++++++------ 2 files changed, 113 insertions(+), 48 deletions(-) diff --git a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp index e32023aeac1e..634a55fec518 100644 --- a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp +++ b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp @@ -185,9 +185,10 @@ struct CGRecordLowering { /// Lowers an ASTRecordLayout to a llvm type. void lower(bool NonVirtualBaseType); void lowerUnion(bool isNoUniqueAddress); - void accumulateFields(); + void accumulateFields(bool isNonVirtualBaseType); RecordDecl::field_iterator - accumulateBitFields(RecordDecl::field_iterator Field, + accumulateBitFields(bool isNonVirtualBaseType, + RecordDecl::field_iterator Field, RecordDecl::field_iterator FieldEnd); void computeVolatileBitfields(); void accumulateBases(); @@ -195,8 +196,10 @@ struct CGRecordLowering { void accumulateVBases(); /// Recursively searches all of the bases to find out if a vbase is /// not the primary vbase of some base class. - bool hasOwnStorage(const CXXRecordDecl *Decl, const CXXRecordDecl *Query); + bool hasOwnStorage(const CXXRecordDecl *Decl, + const CXXRecordDecl *Query) const; void calculateZeroInit(); + CharUnits calculateTailClippingOffset(bool isNonVirtualBaseType) const; /// Lowers bitfield storage types to I8 arrays for bitfields with tail /// padding that is or can potentially be used. void clipTailPadding(); @@ -287,7 +290,7 @@ void CGRecordLowering::lower(bool NVBaseType) { computeVolatileBitfields(); return; } - accumulateFields(); + accumulateFields(NVBaseType); // RD implies C++. if (RD) { accumulateVPtrs(); @@ -378,12 +381,12 @@ void CGRecordLowering::lowerUnion(bool isNoUniqueAddress) { Packed = true; } -void CGRecordLowering::accumulateFields() { +void CGRecordLowering::accumulateFields(bool isNonVirtualBaseType) { for (RecordDecl::field_iterator Field = D->field_begin(), FieldEnd = D->field_end(); Field != FieldEnd;) { if (Field->isBitField()) { - Field = accumulateBitFields(Field, FieldEnd); + Field = accumulateBitFields(isNonVirtualBaseType, Field, FieldEnd); assert((Field == FieldEnd || !Field->isBitField()) && "Failed to accumulate all the bitfields"); } else if (Field->isZeroSize(Context)) { @@ -404,9 +407,12 @@ void CGRecordLowering::accumulateFields() { } // Create members for bitfields. Field is a bitfield, and FieldEnd is the end -// iterator of the record. Return the first non-bitfield encountered. +// iterator of the record. Return the first non-bitfield encountered. We need +// to know whether this is the base or complete layout, as virtual bases could +// affect the upper bound of bitfield access unit allocation. RecordDecl::field_iterator -CGRecordLowering::accumulateBitFields(RecordDecl::field_iterator Field, +CGRecordLowering::accumulateBitFields(bool isNonVirtualBaseType, + RecordDecl::field_iterator Field, RecordDecl::field_iterator FieldEnd) { if (isDiscreteBitFieldABI()) { // Run stores the first element of the current run of bitfields. FieldEnd is @@ -505,6 +511,10 @@ CGRecordLowering::accumulateBitFields(RecordDecl::field_iterator Field, bitsToCharUnits(Context.getTargetInfo().getRegisterWidth()); unsigned CharBits = Context.getCharWidth(); + // Limit of useable tail padding at end of the record. Computed lazily and + // cached here. + CharUnits ScissorOffset = CharUnits::Zero(); + // Data about the start of the span we're accumulating to create an access // unit from. Begin is the first bitfield of the span. If Begin is FieldEnd, // we've not got a current span. The span starts at the BeginOffset character @@ -630,10 +640,14 @@ CGRecordLowering::accumulateBitFields(RecordDecl::field_iterator Field, LimitOffset = bitsToCharUnits(getFieldBitOffset(*Probe)); goto FoundLimit; } - // We reached the end of the fields. We can't necessarily use tail - // padding in C++ structs, so the NonVirtual size is what we must - // use there. - LimitOffset = RD ? Layout.getNonVirtualSize() : Layout.getDataSize(); + // We reached the end of the fields, determine the bounds of useable + // tail padding. As this can be complex for C++, we cache the result. + if (ScissorOffset.isZero()) { + ScissorOffset = calculateTailClippingOffset(isNonVirtualBaseType); + assert(!ScissorOffset.isZero() && "Tail clipping at zero"); + } + + LimitOffset = ScissorOffset; FoundLimit:; CharUnits TypeSize = getSize(Type); @@ -838,13 +852,17 @@ void CGRecordLowering::accumulateVPtrs() { llvm::PointerType::getUnqual(Types.getLLVMContext()))); } -void CGRecordLowering::accumulateVBases() { +CharUnits +CGRecordLowering::calculateTailClippingOffset(bool isNonVirtualBaseType) const { + if (!RD) + return Layout.getDataSize(); + CharUnits ScissorOffset = Layout.getNonVirtualSize(); // In the itanium ABI, it's possible to place a vbase at a dsize that is // smaller than the nvsize. Here we check to see if such a base is placed // before the nvsize and set the scissor offset to that, instead of the // nvsize. - if (isOverlappingVBaseABI()) + if (!isNonVirtualBaseType && isOverlappingVBaseABI()) for (const auto &Base : RD->vbases()) { const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); if (BaseDecl->isEmpty()) @@ -856,8 +874,13 @@ void CGRecordLowering::accumulateVBases() { ScissorOffset = std::min(ScissorOffset, Layout.getVBaseClassOffset(BaseDecl)); } - Members.push_back(MemberInfo(ScissorOffset, MemberInfo::Scissor, nullptr, - RD)); + + return ScissorOffset; +} + +void CGRecordLowering::accumulateVBases() { + Members.push_back(MemberInfo(calculateTailClippingOffset(false), + MemberInfo::Scissor, nullptr, RD)); for (const auto &Base : RD->vbases()) { const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); if (BaseDecl->isEmpty()) @@ -882,7 +905,7 @@ void CGRecordLowering::accumulateVBases() { } bool CGRecordLowering::hasOwnStorage(const CXXRecordDecl *Decl, - const CXXRecordDecl *Query) { + const CXXRecordDecl *Query) const { const ASTRecordLayout &DeclLayout = Context.getASTRecordLayout(Decl); if (DeclLayout.isPrimaryBaseVirtual() && DeclLayout.getPrimaryBase() == Query) return false; diff --git a/clang/test/CodeGenCXX/bitfield-access-tail.cpp b/clang/test/CodeGenCXX/bitfield-access-tail.cpp index 68716fdf3b1d..1539e17cad43 100644 --- a/clang/test/CodeGenCXX/bitfield-access-tail.cpp +++ b/clang/test/CodeGenCXX/bitfield-access-tail.cpp @@ -2,45 +2,45 @@ // Configs that have cheap unaligned access // Little Endian -// RUN: %clang_cc1 -triple=aarch64-apple-darwin %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=aarch64-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s +// RUN: %clang_cc1 -triple=aarch64-apple-darwin %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s +// RUN: %clang_cc1 -triple=aarch64-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s // RUN: %clang_cc1 -triple=arm-apple-darwin %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT-DWN32 %s -// RUN: %clang_cc1 -triple=arm-none-eabi %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=i686-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=loongarch64-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=powerpcle-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=ve-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=wasm32 %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=wasm64 %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=x86_64-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s +// RUN: %clang_cc1 -triple=arm-none-eabi %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=i686-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=loongarch64-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s +// RUN: %clang_cc1 -triple=powerpcle-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=ve-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s +// RUN: %clang_cc1 -triple=wasm32 %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=wasm64 %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s +// RUN: %clang_cc1 -triple=x86_64-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s // Big Endian -// RUN: %clang_cc1 -triple=powerpc-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=powerpc64-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=systemz %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s +// RUN: %clang_cc1 -triple=powerpc-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=powerpc64-linux-gnu %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s +// RUN: %clang_cc1 -triple=systemz %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s // Configs that have expensive unaligned access // Little Endian -// RUN: %clang_cc1 -triple=amdgcn-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=arc-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=bpf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=csky %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=hexagon-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=le64-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=loongarch32-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=nvptx-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=riscv32 %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=riscv64 %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=spir-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=xcore-none-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s +// RUN: %clang_cc1 -triple=amdgcn-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s +// RUN: %clang_cc1 -triple=arc-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=bpf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s +// RUN: %clang_cc1 -triple=csky %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=hexagon-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=le64-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s +// RUN: %clang_cc1 -triple=loongarch32-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=nvptx-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=riscv32 %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=riscv64 %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s +// RUN: %clang_cc1 -triple=spir-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=xcore-none-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s // Big endian -// RUN: %clang_cc1 -triple=lanai-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=m68k-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=mips-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=mips64-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=sparc-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s -// RUN: %clang_cc1 -triple=tce-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s +// RUN: %clang_cc1 -triple=lanai-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=m68k-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=mips-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=mips64-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT64 %s +// RUN: %clang_cc1 -triple=sparc-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s +// RUN: %clang_cc1 -triple=tce-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT,LAYOUT32 %s // Can use tail padding struct Pod { @@ -113,3 +113,45 @@ struct __attribute__((packed)) PNonPod { // LAYOUT-DWN32-NEXT: + +struct __attribute__((aligned(4))) Empty {} empty; + +struct Char { char a; } cbase; +struct D : virtual Char { + [[no_unique_address]] Empty e0; + [[no_unique_address]] Empty e1; + unsigned a : 24; // keep as 24bits +} d; +// CHECK-LABEL: LLVMType:%struct.D = +// LAYOUT64-SAME: type <{ ptr, [3 x i8], %struct.Char, [4 x i8] }> +// LAYOUT32-SAME: type { ptr, [3 x i8], %struct.Char } +// LAYOUT-DWN32-SAME: type { ptr, [3 x i8], %struct.Char } +// CHECK-NEXT: NonVirtualBaseLLVMType: +// LAYOUT64-SAME: %struct.D.base = type <{ ptr, i32 }> +// LAYOUT32-SAME: %struct.D = type { ptr, [3 x i8], %struct.Char } +// LAYOUT-DWN32-SAME: %struct.D = type { ptr, [3 x i8], %struct.Char } +// CHECK: BitFields:[ +// LAYOUT-NEXT: + +struct Int { int a; } ibase; +struct E : virtual Int { + [[no_unique_address]] Empty e0; + [[no_unique_address]] Empty e1; + unsigned a : 24; // expand to 32 +} e; +// CHECK-LABEL: LLVMType:%struct.E = +// LAYOUT64-SAME: type <{ ptr, i32, %struct.Int }> +// LAYOUT32-SAME: type { ptr, i32, %struct.Int } +// LAYOUT-DWN32-SAME: type { ptr, i32, %struct.Int } +// CHECK-NEXT: NonVirtualBaseLLVMType:%struct.E.base = +// LAYOUT64-SAME: type <{ ptr, i32 }> +// LAYOUT32-SAME: type { ptr, i32 } +// LAYOUT-DWN32-SAME: type { ptr, i32 } +// CHECK: BitFields:[ +// LAYOUT-NEXT: -- GitLab From ed6edf262d9061ce3c024754c4981299b5184ee2 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Mon, 1 Apr 2024 13:35:29 -0700 Subject: [PATCH 0135/1690] [scudo] Change isPowerOfTwo macro to return false for zero. (#87120) Clean-up all of the calls and remove the redundant == 0 checks. There is only one small visible change. For non-Android, the memalign function will now fail if alignment is zero. Before this would have passed. --- compiler-rt/lib/scudo/standalone/common.h | 6 +++++- compiler-rt/lib/scudo/standalone/stack_depot.h | 4 ++-- compiler-rt/lib/scudo/standalone/wrappers_c_checks.h | 6 ++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h index ae45683f1ee3..151fbd317e74 100644 --- a/compiler-rt/lib/scudo/standalone/common.h +++ b/compiler-rt/lib/scudo/standalone/common.h @@ -28,7 +28,11 @@ template inline Dest bit_cast(const Source &S) { return D; } -inline constexpr bool isPowerOfTwo(uptr X) { return (X & (X - 1)) == 0; } +inline constexpr bool isPowerOfTwo(uptr X) { + if (X == 0) + return false; + return (X & (X - 1)) == 0; +} inline constexpr uptr roundUp(uptr X, uptr Boundary) { DCHECK(isPowerOfTwo(Boundary)); diff --git a/compiler-rt/lib/scudo/standalone/stack_depot.h b/compiler-rt/lib/scudo/standalone/stack_depot.h index 98cd9707a646..0176c40aa899 100644 --- a/compiler-rt/lib/scudo/standalone/stack_depot.h +++ b/compiler-rt/lib/scudo/standalone/stack_depot.h @@ -103,7 +103,7 @@ public: // Ensure that RingSize, RingMask and TabMask are set up in a way that // all accesses are within range of BufSize. bool isValid(uptr BufSize) const { - if (RingSize == 0 || !isPowerOfTwo(RingSize)) + if (!isPowerOfTwo(RingSize)) return false; uptr RingBytes = sizeof(atomic_u64) * RingSize; if (RingMask + 1 != RingSize) @@ -112,7 +112,7 @@ public: if (TabMask == 0) return false; uptr TabSize = TabMask + 1; - if (TabSize == 0 || !isPowerOfTwo(TabSize)) + if (!isPowerOfTwo(TabSize)) return false; uptr TabBytes = sizeof(atomic_u32) * TabSize; diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c_checks.h b/compiler-rt/lib/scudo/standalone/wrappers_c_checks.h index 9cd48e82792e..d0288699cf1b 100644 --- a/compiler-rt/lib/scudo/standalone/wrappers_c_checks.h +++ b/compiler-rt/lib/scudo/standalone/wrappers_c_checks.h @@ -31,15 +31,13 @@ inline void *setErrnoOnNull(void *Ptr) { // Checks aligned_alloc() parameters, verifies that the alignment is a power of // two and that the size is a multiple of alignment. inline bool checkAlignedAllocAlignmentAndSize(uptr Alignment, uptr Size) { - return Alignment == 0 || !isPowerOfTwo(Alignment) || - !isAligned(Size, Alignment); + return !isPowerOfTwo(Alignment) || !isAligned(Size, Alignment); } // Checks posix_memalign() parameters, verifies that alignment is a power of two // and a multiple of sizeof(void *). inline bool checkPosixMemalignAlignment(uptr Alignment) { - return Alignment == 0 || !isPowerOfTwo(Alignment) || - !isAligned(Alignment, sizeof(void *)); + return !isPowerOfTwo(Alignment) || !isAligned(Alignment, sizeof(void *)); } // Returns true if calloc(Size, N) overflows on Size*N calculation. Use a -- GitLab From e93b5f5a4776ffea12d03652559dfdf8d421184c Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Mon, 1 Apr 2024 13:05:34 -0700 Subject: [PATCH 0136/1690] [ubsan][NFC] Remove recently added `cl::init(false)` Extracted from #84858 --- clang/lib/CodeGen/BackendUtil.cpp | 7 +++---- clang/lib/CodeGen/CGExpr.cpp | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 82b30b8d8156..1220c575d1df 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -101,20 +101,19 @@ namespace llvm { extern cl::opt PrintPipelinePasses; cl::opt ClRemoveTraps("clang-remove-traps", cl::Optional, - cl::desc("Insert remove-traps pass."), - cl::init(false)); + cl::desc("Insert remove-traps pass.")); // Experiment to move sanitizers earlier. static cl::opt ClSanitizeOnOptimizerEarlyEP( "sanitizer-early-opt-ep", cl::Optional, - cl::desc("Insert sanitizers on OptimizerEarlyEP."), cl::init(false)); + cl::desc("Insert sanitizers on OptimizerEarlyEP.")); extern cl::opt ProfileCorrelate; // Re-link builtin bitcodes after optimization cl::opt ClRelinkBuiltinBitcodePostop( "relink-builtin-bitcode-postop", cl::Optional, - cl::desc("Re-link builtin bitcodes after optimization."), cl::init(false)); + cl::desc("Re-link builtin bitcodes after optimization.")); } // namespace llvm namespace { diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index e0d5575d57d0..54432353e742 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -56,8 +56,7 @@ using namespace CodeGen; // Experiment to make sanitizers easier to debug static llvm::cl::opt ClSanitizeDebugDeoptimization( "ubsan-unique-traps", llvm::cl::Optional, - llvm::cl::desc("Deoptimize traps for UBSAN so there is 1 trap per check"), - llvm::cl::init(false)); + llvm::cl::desc("Deoptimize traps for UBSAN so there is 1 trap per check")); //===--------------------------------------------------------------------===// // Miscellaneous Helper Methods -- GitLab From b8cc3ba409dc850776f37e27613bf74f5a80d66a Mon Sep 17 00:00:00 2001 From: Lei Wang Date: Mon, 1 Apr 2024 13:54:54 -0700 Subject: [PATCH 0137/1690] [PseudoProbe] Extend to skip instrumenting probe into the dests of invoke (#79919) As before we only skip instrumenting probe of `unwind`(`KnownColdBlock`) block, this PR extends to skip the both EH flow from `invoke`, i.e. also skip the `normal` dest. For more contexts: when doing call-to-invoke conversion, the block is split by the `invoke` and two extra blocks(`normal` and `unwind`) are added. With this PR, the instrumentation is the same as the one before the call-to-invoke conversion. One significant benefit is this can help mitigate the "unstable IR" issue(https://discourse.llvm.org/t/ipo-for-linkonce-odr-functions/69404), the two versions now are on the same probe instrumentation, expected to be the same checksum. To achieve the same checksum, some tweaks is needed: - Now it also skips incrementing the probe ID for the skipped probe. - The checksum is also computed based on the CFG that skips the EH edges. We observed this fixes ~5% mismatched samples. --- llvm/include/llvm/Analysis/EHUtils.h | 1 - .../llvm/Transforms/IPO/SampleProfileProbe.h | 13 +- .../lib/Transforms/IPO/SampleProfileProbe.cpp | 121 ++++++++++++-- .../ThinLTO/X86/pseudo-probe-desc-import.ll | 4 +- .../SampleProfile/pseudo-probe-eh.ll | 2 +- .../SampleProfile/pseudo-probe-invoke.ll | 155 ++++++++++++++++++ 6 files changed, 276 insertions(+), 20 deletions(-) create mode 100644 llvm/test/Transforms/SampleProfile/pseudo-probe-invoke.ll diff --git a/llvm/include/llvm/Analysis/EHUtils.h b/llvm/include/llvm/Analysis/EHUtils.h index f2ff6cbd2e90..3ad0878bd64f 100644 --- a/llvm/include/llvm/Analysis/EHUtils.h +++ b/llvm/include/llvm/Analysis/EHUtils.h @@ -79,7 +79,6 @@ static void computeEHOnlyBlocks(FunctionT &F, DenseSet &EHBlocks) { } } - EHBlocks.clear(); for (auto Entry : Statuses) { if (Entry.second == EH) EHBlocks.insert(Entry.first); diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h index 0f2729a9462d..03aa93ce6bd3 100644 --- a/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h +++ b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h @@ -81,8 +81,17 @@ private: uint64_t getFunctionHash() const { return FunctionHash; } uint32_t getBlockId(const BasicBlock *BB) const; uint32_t getCallsiteId(const Instruction *Call) const; - void computeCFGHash(); - void computeProbeIdForBlocks(); + void findUnreachableBlocks(DenseSet &BlocksToIgnore); + void findInvokeNormalDests(DenseSet &InvokeNormalDests); + void computeBlocksToIgnore(DenseSet &BlocksToIgnore, + DenseSet &BlocksAndCallsToIgnore); + void computeProbeIdForCallsites( + const DenseSet &BlocksAndCallsToIgnore); + const Instruction * + getOriginalTerminator(const BasicBlock *Head, + const DenseSet &BlocksToIgnore); + void computeCFGHash(const DenseSet &BlocksToIgnore); + void computeProbeIdForBlocks(const DenseSet &BlocksToIgnore); void computeProbeIdForCallsites(); Function *F; diff --git a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp index 090e5560483e..4d0fa24bd57c 100644 --- a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp @@ -173,21 +173,114 @@ SampleProfileProber::SampleProfileProber(Function &Func, BlockProbeIds.clear(); CallProbeIds.clear(); LastProbeId = (uint32_t)PseudoProbeReservedId::Last; - computeProbeIdForBlocks(); - computeProbeIdForCallsites(); - computeCFGHash(); + + DenseSet BlocksToIgnore; + DenseSet BlocksAndCallsToIgnore; + computeBlocksToIgnore(BlocksToIgnore, BlocksAndCallsToIgnore); + + computeProbeIdForBlocks(BlocksToIgnore); + computeProbeIdForCallsites(BlocksAndCallsToIgnore); + computeCFGHash(BlocksToIgnore); +} + +// Two purposes to compute the blocks to ignore: +// 1. Reduce the IR size. +// 2. Make the instrumentation(checksum) stable. e.g. the frondend may +// generate unstable IR while optimizing nounwind attribute, some versions are +// optimized with the call-to-invoke conversion, while other versions do not. +// This discrepancy in probe ID could cause profile mismatching issues. +// Note that those ignored blocks are either cold blocks or new split blocks +// whose original blocks are instrumented, so it shouldn't degrade the profile +// quality. +void SampleProfileProber::computeBlocksToIgnore( + DenseSet &BlocksToIgnore, + DenseSet &BlocksAndCallsToIgnore) { + // Ignore the cold EH and unreachable blocks and calls. + computeEHOnlyBlocks(*F, BlocksAndCallsToIgnore); + findUnreachableBlocks(BlocksAndCallsToIgnore); + + BlocksToIgnore.insert(BlocksAndCallsToIgnore.begin(), + BlocksAndCallsToIgnore.end()); + + // Handle the call-to-invoke conversion case: make sure that the probe id and + // callsite id are consistent before and after the block split. For block + // probe, we only keep the head block probe id and ignore the block ids of the + // normal dests. For callsite probe, it's different to block probe, there is + // no additional callsite in the normal dests, so we don't ignore the + // callsites. + findInvokeNormalDests(BlocksToIgnore); +} + +// Unreachable blocks and calls are always cold, ignore them. +void SampleProfileProber::findUnreachableBlocks( + DenseSet &BlocksToIgnore) { + for (auto &BB : *F) { + if (&BB != &F->getEntryBlock() && pred_size(&BB) == 0) + BlocksToIgnore.insert(&BB); + } +} + +// In call-to-invoke conversion, basic block can be split into multiple blocks, +// only instrument probe in the head block, ignore the normal dests. +void SampleProfileProber::findInvokeNormalDests( + DenseSet &InvokeNormalDests) { + for (auto &BB : *F) { + auto *TI = BB.getTerminator(); + if (auto *II = dyn_cast(TI)) { + auto *ND = II->getNormalDest(); + InvokeNormalDests.insert(ND); + + // The normal dest and the try/catch block are connected by an + // unconditional branch. + while (pred_size(ND) == 1) { + auto *Pred = *pred_begin(ND); + if (succ_size(Pred) == 1) { + InvokeNormalDests.insert(Pred); + ND = Pred; + } else + break; + } + } + } +} + +// The call-to-invoke conversion splits the original block into a list of block, +// we need to compute the hash using the original block's successors to keep the +// CFG Hash consistent. For a given head block, we keep searching the +// succesor(normal dest or unconditional branch dest) to find the tail block, +// the tail block's successors are the original block's successors. +const Instruction *SampleProfileProber::getOriginalTerminator( + const BasicBlock *Head, const DenseSet &BlocksToIgnore) { + auto *TI = Head->getTerminator(); + if (auto *II = dyn_cast(TI)) { + return getOriginalTerminator(II->getNormalDest(), BlocksToIgnore); + } else if (succ_size(Head) == 1 && + BlocksToIgnore.contains(*succ_begin(Head))) { + // Go to the unconditional branch dest. + return getOriginalTerminator(*succ_begin(Head), BlocksToIgnore); + } + return TI; } // Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index // value of each BB in the CFG. The higher 32 bits record the number of edges // preceded by the number of indirect calls. // This is derived from FuncPGOInstrumentation::computeCFGHash(). -void SampleProfileProber::computeCFGHash() { +void SampleProfileProber::computeCFGHash( + const DenseSet &BlocksToIgnore) { std::vector Indexes; JamCRC JC; for (auto &BB : *F) { - for (BasicBlock *Succ : successors(&BB)) { + if (BlocksToIgnore.contains(&BB)) + continue; + + auto *TI = getOriginalTerminator(&BB, BlocksToIgnore); + for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) { + auto *Succ = TI->getSuccessor(I); auto Index = getBlockId(Succ); + // Ingore ignored-block(zero ID) to avoid unstable checksum. + if (Index == 0) + continue; for (int J = 0; J < 4; J++) Indexes.push_back((uint8_t)(Index >> (J * 8))); } @@ -207,23 +300,23 @@ void SampleProfileProber::computeCFGHash() { << ", Hash = " << FunctionHash << "\n"); } -void SampleProfileProber::computeProbeIdForBlocks() { - DenseSet KnownColdBlocks; - computeEHOnlyBlocks(*F, KnownColdBlocks); - // Insert pseudo probe to non-cold blocks only. This will reduce IR size as - // well as the binary size while retaining the profile quality. +void SampleProfileProber::computeProbeIdForBlocks( + const DenseSet &BlocksToIgnore) { for (auto &BB : *F) { - ++LastProbeId; - if (!KnownColdBlocks.contains(&BB)) - BlockProbeIds[&BB] = LastProbeId; + if (BlocksToIgnore.contains(&BB)) + continue; + BlockProbeIds[&BB] = ++LastProbeId; } } -void SampleProfileProber::computeProbeIdForCallsites() { +void SampleProfileProber::computeProbeIdForCallsites( + const DenseSet &BlocksAndCallsToIgnore) { LLVMContext &Ctx = F->getContext(); Module *M = F->getParent(); for (auto &BB : *F) { + if (BlocksAndCallsToIgnore.contains(&BB)) + continue; for (auto &I : BB) { if (!isa(I)) continue; diff --git a/llvm/test/ThinLTO/X86/pseudo-probe-desc-import.ll b/llvm/test/ThinLTO/X86/pseudo-probe-desc-import.ll index 21dd8c0fe924..f915aaccc06e 100644 --- a/llvm/test/ThinLTO/X86/pseudo-probe-desc-import.ll +++ b/llvm/test/ThinLTO/X86/pseudo-probe-desc-import.ll @@ -12,8 +12,8 @@ ; RUN: llvm-lto -thinlto-action=import %t3.bc -thinlto-index=%t3.index.bc -o /dev/null 2>&1 | FileCheck %s --check-prefix=WARN -; CHECK-NOT: {i64 6699318081062747564, i64 4294967295, !"foo" -; CHECK: !{i64 -2624081020897602054, i64 281479271677951, !"main" +; CHECK-NOT: {i64 6699318081062747564, i64 [[#]], !"foo" +; CHECK: !{i64 -2624081020897602054, i64 [[#]], !"main" ; WARN: warning: Pseudo-probe ignored: source module '{{.*}}' is compiled with -fpseudo-probe-for-profiling while destination module '{{.*}}' is not diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-eh.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-eh.ll index 697ef44fb7ed..9954914bca43 100644 --- a/llvm/test/Transforms/SampleProfile/pseudo-probe-eh.ll +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-eh.ll @@ -18,7 +18,7 @@ entry: to label %ret unwind label %lpad ret: -; CHECK: call void @llvm.pseudoprobe +; CHECK-NOT: call void @llvm.pseudoprobe ret void lpad: ; preds = %entry diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-invoke.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-invoke.ll new file mode 100644 index 000000000000..822ab403dee2 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-invoke.ll @@ -0,0 +1,155 @@ +; REQUIRES: x86_64-linux +; RUN: opt < %s -passes=pseudo-probe -S -o - | FileCheck %s + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +$__clang_call_terminate = comdat any + +@x = dso_local global i32 0, align 4, !dbg !0 + +; Function Attrs: mustprogress noinline nounwind uwtable +define dso_local void @_Z3barv() #0 personality ptr @__gxx_personality_v0 !dbg !14 { +entry: +; CHECK: call void @llvm.pseudoprobe(i64 -1069303473483922844, i64 1 + %0 = load volatile i32, ptr @x, align 4, !dbg !17, !tbaa !19 + %tobool = icmp ne i32 %0, 0, !dbg !17 + br i1 %tobool, label %if.then, label %if.else, !dbg !23 + +if.then: ; preds = %entry +; CHECK: call void @llvm.pseudoprobe(i64 -1069303473483922844, i64 2 + invoke void @_Z3foov() + to label %invoke.cont unwind label %terminate.lpad, !dbg !24 + +invoke.cont: ; preds = %if.then +; CHECK-NOT: call void @llvm.pseudoprobe(i64 -1069303473483922844, + invoke void @_Z3bazv() + to label %invoke.cont1 unwind label %terminate.lpad, !dbg !26 + +invoke.cont1: ; preds = %invoke.cont +; CHECK-NOT: call void @llvm.pseudoprobe(i64 -1069303473483922844, + br label %if.end, !dbg !27 + +if.else: ; preds = %entry +; CHECK: call void @llvm.pseudoprobe(i64 -1069303473483922844, i64 3 + invoke void @_Z3foov() + to label %invoke.cont2 unwind label %terminate.lpad, !dbg !28 + +invoke.cont2: ; preds = %if.else +; CHECK-NOT: call void @llvm.pseudoprobe(i64 -1069303473483922844, + br label %if.end + +if.end: ; preds = %invoke.cont2, %invoke.cont1 +; CHECK: call void @llvm.pseudoprobe(i64 -1069303473483922844, i64 4 + invoke void @_Z3foov() + to label %invoke.cont3 unwind label %terminate.lpad, !dbg !29 + +invoke.cont3: ; preds = %if.end +; CHECK-NOT: call void @llvm.pseudoprobe(i64 -1069303473483922844, + %1 = load volatile i32, ptr @x, align 4, !dbg !30, !tbaa !19 + %tobool4 = icmp ne i32 %1, 0, !dbg !30 + br i1 %tobool4, label %if.then5, label %if.end6, !dbg !32 + +if.then5: ; preds = %invoke.cont3 +; CHECK: call void @llvm.pseudoprobe(i64 -1069303473483922844, i64 5 + %2 = load volatile i32, ptr @x, align 4, !dbg !33, !tbaa !19 + %inc = add nsw i32 %2, 1, !dbg !33 + store volatile i32 %inc, ptr @x, align 4, !dbg !33, !tbaa !19 + br label %if.end6, !dbg !35 + +if.end6: ; preds = %if.then5, %invoke.cont3 +; CHECK: call void @llvm.pseudoprobe(i64 -1069303473483922844, i64 6 + ret void, !dbg !36 + +terminate.lpad: ; preds = %if.end, %if.else, %invoke.cont, %if.then +; CHECK-NOT: call void @llvm.pseudoprobe(i64 -1069303473483922844, + %3 = landingpad { ptr, i32 } + catch ptr null, !dbg !24 + %4 = extractvalue { ptr, i32 } %3, 0, !dbg !24 + call void @__clang_call_terminate(ptr %4) #3, !dbg !24 + unreachable, !dbg !24 +} + +; Function Attrs: mustprogress noinline nounwind uwtable +define dso_local void @_Z3foov() #0 !dbg !37 { +entry: + ret void, !dbg !38 +} + +declare i32 @__gxx_personality_v0(...) + +; Function Attrs: noinline noreturn nounwind uwtable +define linkonce_odr hidden void @__clang_call_terminate(ptr noundef %0) #1 comdat { + %2 = call ptr @__cxa_begin_catch(ptr %0) #4 + call void @_ZSt9terminatev() #3 + unreachable +} + +declare ptr @__cxa_begin_catch(ptr) + +declare void @_ZSt9terminatev() + +; Function Attrs: mustprogress noinline nounwind uwtable +define dso_local void @_Z3bazv() #0 !dbg !39 { +entry: + ret void, !dbg !40 +} + +; CHECK: ![[#]] = !{i64 -3270123626113159616, i64 4294967295, !"_Z3bazv"} + +attributes #0 = { mustprogress noinline nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" } +attributes #1 = { noinline noreturn nounwind uwtable "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" } +attributes #2 = { mustprogress noinline norecurse nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" } +attributes #3 = { noreturn nounwind } +attributes #4 = { nounwind } + +!llvm.dbg.cu = !{!2} +!llvm.module.flags = !{!7, !8, !9, !10, !11, !12} +!llvm.ident = !{!13} + +!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) +!1 = distinct !DIGlobalVariable(name: "x", scope: !2, file: !3, line: 1, type: !5, isLocal: false, isDefinition: true) +!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, producer: "clang version 19.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, nameTableKind: None) +!3 = !DIFile(filename: "test.cpp", directory: "/home", checksumkind: CSK_MD5, checksum: "a4c7b0392f3fd9c8ebb85065159dbb02") +!4 = !{!0} +!5 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !6) +!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!7 = !{i32 7, !"Dwarf Version", i32 5} +!8 = !{i32 2, !"Debug Info Version", i32 3} +!9 = !{i32 1, !"wchar_size", i32 4} +!10 = !{i32 8, !"PIC Level", i32 2} +!11 = !{i32 7, !"PIE Level", i32 2} +!12 = !{i32 7, !"uwtable", i32 2} +!13 = !{!"clang version 19.0.0"} +!14 = distinct !DISubprogram(name: "bar", linkageName: "_Z3barv", scope: !3, file: !3, line: 4, type: !15, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2) +!15 = !DISubroutineType(types: !16) +!16 = !{null} +!17 = !DILocation(line: 5, column: 6, scope: !18) +!18 = distinct !DILexicalBlock(scope: !14, file: !3, line: 5, column: 6) +!19 = !{!20, !20, i64 0} +!20 = !{!"int", !21, i64 0} +!21 = !{!"omnipotent char", !22, i64 0} +!22 = !{!"Simple C++ TBAA"} +!23 = !DILocation(line: 5, column: 6, scope: !14) +!24 = !DILocation(line: 6, column: 5, scope: !25) +!25 = distinct !DILexicalBlock(scope: !18, file: !3, line: 5, column: 9) +!26 = !DILocation(line: 7, column: 5, scope: !25) +!27 = !DILocation(line: 8, column: 3, scope: !25) +!28 = !DILocation(line: 9, column: 5, scope: !18) +!29 = !DILocation(line: 11, column: 3, scope: !14) +!30 = !DILocation(line: 12, column: 6, scope: !31) +!31 = distinct !DILexicalBlock(scope: !14, file: !3, line: 12, column: 6) +!32 = !DILocation(line: 12, column: 6, scope: !14) +!33 = !DILocation(line: 13, column: 5, scope: !34) +!34 = distinct !DILexicalBlock(scope: !31, file: !3, line: 12, column: 9) +!35 = !DILocation(line: 14, column: 5, scope: !34) +!36 = !DILocation(line: 17, column: 1, scope: !14) +!37 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !3, file: !3, line: 19, type: !15, scopeLine: 19, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2) +!38 = !DILocation(line: 19, column: 13, scope: !37) +!39 = distinct !DISubprogram(name: "baz", linkageName: "_Z3bazv", scope: !3, file: !3, line: 18, type: !15, scopeLine: 18, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2) +!40 = !DILocation(line: 18, column: 13, scope: !39) +!41 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 22, type: !42, scopeLine: 22, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2) +!42 = !DISubroutineType(types: !43) +!43 = !{!6} +!44 = !DILocation(line: 23, column: 3, scope: !41) +!45 = !DILocation(line: 24, column: 1, scope: !41) -- GitLab From f2f01f6b03aa81d5bdbf841a88f8853620c6902b Mon Sep 17 00:00:00 2001 From: Jeff Niu Date: Mon, 1 Apr 2024 13:59:53 -0700 Subject: [PATCH 0138/1690] [llvm][Support] Use `thread_local` caching for llvm::get_threadid() query on Apple systems (#87219) I was profiling our compiler and noticed that `llvm::get_threadid` was at the top of the hotlist, taking up a surprising 5% (7 seconds) in the profile trace. It seems that computing this on MacOS systems is non-trivial, so cache the result in a thread_local. Co-authored-by: Mehdi Amini --- llvm/lib/Support/Unix/Threading.inc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Support/Unix/Threading.inc b/llvm/lib/Support/Unix/Threading.inc index 55e7dcfa4678..839c00c5ebbf 100644 --- a/llvm/lib/Support/Unix/Threading.inc +++ b/llvm/lib/Support/Unix/Threading.inc @@ -115,8 +115,11 @@ uint64_t llvm::get_threadid() { // Calling "mach_thread_self()" bumps the reference count on the thread // port, so we need to deallocate it. mach_task_self() doesn't bump the ref // count. - thread_port_t Self = mach_thread_self(); - mach_port_deallocate(mach_task_self(), Self); + static thread_local thread_port_t Self = [] { + thread_port_t InitSelf = mach_thread_self(); + mach_port_deallocate(mach_task_self(), Self); + return InitSelf; + }(); return Self; #elif defined(__FreeBSD__) return uint64_t(pthread_getthreadid_np()); -- GitLab From a6caceed8d27d4ebd44c517c3114a36a64ebddfe Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Mon, 1 Apr 2024 16:02:12 -0500 Subject: [PATCH 0139/1690] [lldb] Don't crash when attempting to parse breakpoint id `N.` as `N.*` (#87263) We check if the next character after `N.` is `*` before we check its length. Using `split` on the string is cleaner and less error prone than using indices with `find` and `substr`. Note: this does not make `N.` mean anything, it just prevents assertion failures. `N.` is treated the same as an unrecognized breakpoint name: ``` (lldb) breakpoint enable 1 1 breakpoints enabled. (lldb) breakpoint enable 1.* 1 breakpoints enabled. (lldb) breakpoint enable 1. 0 breakpoints enabled. (lldb) breakpoint enable xyz 0 breakpoints enabled. ``` Found via LLDB fuzzers. --- lldb/source/Breakpoint/BreakpointIDList.cpp | 48 +++++++++---------- .../TestBreakpointLocations.py | 6 +++ 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/lldb/source/Breakpoint/BreakpointIDList.cpp b/lldb/source/Breakpoint/BreakpointIDList.cpp index 851d074e7535..97af1d40eb7a 100644 --- a/lldb/source/Breakpoint/BreakpointIDList.cpp +++ b/lldb/source/Breakpoint/BreakpointIDList.cpp @@ -16,6 +16,7 @@ #include "lldb/Utility/StreamString.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" using namespace lldb; using namespace lldb_private; @@ -111,32 +112,27 @@ llvm::Error BreakpointIDList::FindAndReplaceIDRanges( } else { // See if user has specified id.* llvm::StringRef tmp_str = old_args[i].ref(); - size_t pos = tmp_str.find('.'); - if (pos != llvm::StringRef::npos) { - llvm::StringRef bp_id_str = tmp_str.substr(0, pos); - if (BreakpointID::IsValidIDExpression(bp_id_str) && - tmp_str[pos + 1] == '*' && tmp_str.size() == (pos + 2)) { - - BreakpointSP breakpoint_sp; - auto bp_id = BreakpointID::ParseCanonicalReference(bp_id_str); - if (bp_id) - breakpoint_sp = target->GetBreakpointByID(bp_id->GetBreakpointID()); - if (!breakpoint_sp) { - new_args.Clear(); - return llvm::createStringError( - llvm::inconvertibleErrorCode(), - "'%d' is not a valid breakpoint ID.\n", - bp_id->GetBreakpointID()); - } - const size_t num_locations = breakpoint_sp->GetNumLocations(); - for (size_t j = 0; j < num_locations; ++j) { - BreakpointLocation *bp_loc = - breakpoint_sp->GetLocationAtIndex(j).get(); - StreamString canonical_id_str; - BreakpointID::GetCanonicalReference( - &canonical_id_str, bp_id->GetBreakpointID(), bp_loc->GetID()); - new_args.AppendArgument(canonical_id_str.GetString()); - } + auto [prefix, suffix] = tmp_str.split('.'); + if (suffix == "*" && BreakpointID::IsValidIDExpression(prefix)) { + + BreakpointSP breakpoint_sp; + auto bp_id = BreakpointID::ParseCanonicalReference(prefix); + if (bp_id) + breakpoint_sp = target->GetBreakpointByID(bp_id->GetBreakpointID()); + if (!breakpoint_sp) { + new_args.Clear(); + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "'%d' is not a valid breakpoint ID.\n", + bp_id->GetBreakpointID()); + } + const size_t num_locations = breakpoint_sp->GetNumLocations(); + for (size_t j = 0; j < num_locations; ++j) { + BreakpointLocation *bp_loc = + breakpoint_sp->GetLocationAtIndex(j).get(); + StreamString canonical_id_str; + BreakpointID::GetCanonicalReference( + &canonical_id_str, bp_id->GetBreakpointID(), bp_loc->GetID()); + new_args.AppendArgument(canonical_id_str.GetString()); } } } diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_locations/TestBreakpointLocations.py b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/TestBreakpointLocations.py index 8930bea619bb..d87e6275f7b5 100644 --- a/lldb/test/API/functionalities/breakpoint/breakpoint_locations/TestBreakpointLocations.py +++ b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/TestBreakpointLocations.py @@ -293,6 +293,12 @@ class BreakpointLocationsTestCase(TestBase): startstr="3 breakpoints enabled.", ) + # The 'breakpoint enable 1.' command should not crash. + self.expect( + "breakpoint enable 1.", + startstr="0 breakpoints enabled.", + ) + # The 'breakpoint disable 1.1' command should disable 1 location. self.expect( "breakpoint disable 1.1", -- GitLab From 03577ced1f55bf96224513f2414bf025d6877fac Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Mon, 1 Apr 2024 14:11:02 -0700 Subject: [PATCH 0140/1690] [BOLT][NFC] Fix typo --- bolt/include/bolt/Core/BinaryFunction.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h index 5089f8491280..bc047fefa315 100644 --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -1168,7 +1168,7 @@ public: /// Pass an offset of the entry point in the input binary and a corresponding /// global symbol to the callback function. /// - /// Return true of all callbacks returned true, false otherwise. + /// Return true if all callbacks returned true, false otherwise. bool forEachEntryPoint(EntryPointCallbackTy Callback) const; /// Return MC symbol associated with the end of the function. -- GitLab From 70e189fbc96909d3841dd2bca4a2909345cd826f Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Mon, 1 Apr 2024 14:13:56 -0700 Subject: [PATCH 0141/1690] [libc] fixup ftello test (#87282) Use a seek offset that fits within the file size. This was missed in presubmit because the FILE based stdio tests aren't run in overlay mode; fullbuild is not tested in presubmit. WRITE_SIZE == 11, so using a value of 42 for offseto would cause the expression `WRITE_SIZE - offseto` to evaluate to -31 as an unsigned 64b integer (18446744073709551585ULL). Fixes #86928 --- libc/test/src/stdio/ftell_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/test/src/stdio/ftell_test.cpp b/libc/test/src/stdio/ftell_test.cpp index 68a969ed0c30..62745e2194be 100644 --- a/libc/test/src/stdio/ftell_test.cpp +++ b/libc/test/src/stdio/ftell_test.cpp @@ -39,7 +39,7 @@ protected: // still return the correct effective offset. ASSERT_EQ(size_t(LIBC_NAMESPACE::ftell(file)), WRITE_SIZE); - off_t offseto = 42; + off_t offseto = 5; ASSERT_EQ(0, LIBC_NAMESPACE::fseeko(file, offseto, SEEK_SET)); ASSERT_EQ(LIBC_NAMESPACE::ftello(file), offseto); ASSERT_EQ(0, LIBC_NAMESPACE::fseeko(file, -offseto, SEEK_END)); -- GitLab From 6b136ce738d1acc96d926d7999419867dea16961 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 1 Apr 2024 14:35:39 -0700 Subject: [PATCH 0142/1690] [workflows] issue-write: Exit early if there are no comments (#87114) This will eliminate some unnecessary REST API calls. --- .github/workflows/issue-write.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml index 02a5f7c213e8..f5b84fec17a7 100644 --- a/.github/workflows/issue-write.yml +++ b/.github/workflows/issue-write.yml @@ -31,7 +31,7 @@ jobs: script: | var fs = require('fs'); const comments = JSON.parse(fs.readFileSync('./comments')); - if (!comments) { + if (!comments || comments.length == 0) { return; } -- GitLab From 0478adc97e1a4018d866520cb149b6e6c2a9101a Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Mon, 1 Apr 2024 14:58:28 -0700 Subject: [PATCH 0143/1690] [Object,ELFTypes] Remove TargetEndianness Finish the rename by #86604 --- llvm/include/llvm/Object/ELFTypes.h | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h index 4617b70a2f12..4ab23e4ea81b 100644 --- a/llvm/include/llvm/Object/ELFTypes.h +++ b/llvm/include/llvm/Object/ELFTypes.h @@ -51,7 +51,6 @@ private: using packed = support::detail::packed_endian_specific_integral; public: - static const endianness TargetEndianness = E; static const endianness Endianness = E; static const bool Is64Bits = Is64; -- GitLab From 1d5e5f4d3c68e63ced47ee9b17d62fb995aa1e62 Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Mon, 1 Apr 2024 15:06:10 -0700 Subject: [PATCH 0144/1690] [GISEL][NFC] Fix comment for widenScalarToNextPow2 The docstring for this function incorrectly specified when a widening is not performed. This patch adds the additional specification for what happens when the type size is a power of two but it is less than MinSize. --- llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index 6afaea3f3fc5..82e713f30ea3 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -879,7 +879,8 @@ public: } /// Widen the scalar to the next power of two that is at least MinSize. - /// No effect if the type is not a scalar or is a power of two. + /// No effect if the type is a power of two, except if the type is smaller + /// than MinSize, or if the type is a vector type. LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize = 0) { using namespace LegalityPredicates; -- GitLab From 1e15371dd8843dfc52b9435afaa133997c1773d8 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 1 Apr 2024 15:14:49 -0700 Subject: [PATCH 0145/1690] [ThinLTO][TypeProf] Implement vtable def import (#79381) Add annotated vtable GUID as referenced variables in per function summary, and update bitcode writer to create value-ids for these referenced vtables. - This is the part3 of type profiling work, and described in the "Virtual Table Definition Import" [1] section of the RFC. [1] https://github.com/llvm/llvm-project/pull/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 --- llvm/include/llvm/ProfileData/InstrProf.h | 12 +++- .../IndirectCallPromotionAnalysis.cpp | 4 ++ llvm/lib/Analysis/ModuleSummaryAnalysis.cpp | 20 ++++++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 13 +++- llvm/lib/ProfileData/InstrProf.cpp | 70 +++++++++++++------ .../thinlto-func-summary-vtableref-pgo.ll | 37 ++++++---- 6 files changed, 120 insertions(+), 36 deletions(-) diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index fd66c4ed948f..eb3c10bcba1c 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -283,7 +283,7 @@ void annotateValueSite(Module &M, Instruction &Inst, /// Extract the value profile data from \p Inst which is annotated with /// value profile meta data. Return false if there is no value data annotated, -/// otherwise return true. +/// otherwise return true. bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, @@ -291,6 +291,16 @@ bool getValueProfDataFromInst(const Instruction &Inst, uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue = false); +/// Extract the value profile data from \p Inst and returns them if \p Inst is +/// annotated with value profile data. Returns nullptr otherwise. It's similar +/// to `getValueProfDataFromInst` above except that an array is allocated only +/// after a preliminary checking that the value profiles of kind `ValueKind` +/// exist. +std::unique_ptr +getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, uint32_t &ActualNumValueData, + uint64_t &TotalC, bool GetNoICPValue = false); + inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; } /// Return the PGOFuncName meta data associated with a function. diff --git a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp index ebfa1c8fc08e..ab53717eb889 100644 --- a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp +++ b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp @@ -45,6 +45,10 @@ static cl::opt cl::desc("Max number of promotions for a single indirect " "call callsite")); +cl::opt MaxNumVTableAnnotations( + "icp-max-num-vtables", cl::init(6), cl::Hidden, + cl::desc("Max number of vtables annotated for a vtable load instruction.")); + ICallPromotionAnalysis::ICallPromotionAnalysis() { ValueDataArray = std::make_unique(MaxNumPromotions); } diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index 1f15e9478324..3ad0bab827a5 100644 --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -82,6 +82,8 @@ static cl::opt ModuleSummaryDotFile( extern cl::opt ScalePartialSampleProfileWorkingSetSize; +extern cl::opt MaxNumVTableAnnotations; + // Walk through the operands of a given User via worklist iteration and populate // the set of GlobalValue references encountered. Invoked either on an // Instruction or a GlobalVariable (which walks its initializer). @@ -124,6 +126,24 @@ static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser, Worklist.push_back(Operand); } } + + const Instruction *I = dyn_cast(CurUser); + if (I) { + uint32_t ActualNumValueData = 0; + uint64_t TotalCount = 0; + // MaxNumVTableAnnotations is the maximum number of vtables annotated on + // the instruction. + auto ValueDataArray = + getValueProfDataFromInst(*I, IPVK_VTableTarget, MaxNumVTableAnnotations, + ActualNumValueData, TotalCount); + + if (ValueDataArray.get()) { + for (uint32_t j = 0; j < ActualNumValueData; j++) { + RefEdges.insert(Index.getOrInsertValueInfo(/* VTableGUID = */ + ValueDataArray[j].Value)); + } + } + } return HasBlockAddress; } diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 221eeaae6e2b..dd554e422516 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -203,7 +203,7 @@ public: for (const auto &GUIDSummaryLists : *Index) // Examine all summaries for this GUID. for (auto &Summary : GUIDSummaryLists.second.SummaryList) - if (auto FS = dyn_cast(Summary.get())) + if (auto FS = dyn_cast(Summary.get())) { // For each call in the function summary, see if the call // is to a GUID (which means it is for an indirect call, // otherwise we would have a Value for it). If so, synthesize @@ -211,6 +211,15 @@ public: for (auto &CallEdge : FS->calls()) if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue()) assignValueId(CallEdge.first.getGUID()); + + // For each referenced variables in the function summary, see if the + // variable is represented by a GUID (as opposed to a symbol to + // declarations or definitions in the module). If so, synthesize a + // value id. + for (auto &RefEdge : FS->refs()) + if (!RefEdge.haveGVs() || !RefEdge.getValue()) + assignValueId(RefEdge.getGUID()); + } } protected: @@ -4188,7 +4197,7 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( NameVals.push_back(SpecialRefCnts.second); // worefcnt for (auto &RI : FS->refs()) - NameVals.push_back(VE.getValueID(RI.getValue())); + NameVals.push_back(getValueId(RI)); const bool UseRelBFRecord = WriteRelBFToSummary && !F.hasProfileData() && diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 90c3cfc45b98..95f900d0fff1 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -1271,46 +1271,44 @@ void annotateValueSite(Module &M, Instruction &Inst, Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals)); } -bool getValueProfDataFromInst(const Instruction &Inst, - InstrProfValueKind ValueKind, - uint32_t MaxNumValueData, - InstrProfValueData ValueData[], - uint32_t &ActualNumValueData, uint64_t &TotalC, - bool GetNoICPValue) { +MDNode *mayHaveValueProfileOfKind(const Instruction &Inst, + InstrProfValueKind ValueKind) { MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof); if (!MD) - return false; + return nullptr; - unsigned NOps = MD->getNumOperands(); + if (MD->getNumOperands() < 5) + return nullptr; - if (NOps < 5) - return false; - - // Operand 0 is a string tag "VP": MDString *Tag = cast(MD->getOperand(0)); - if (!Tag) - return false; - - if (!Tag->getString().equals("VP")) - return false; + if (!Tag || !Tag->getString().equals("VP")) + return nullptr; // Now check kind: ConstantInt *KindInt = mdconst::dyn_extract(MD->getOperand(1)); if (!KindInt) - return false; + return nullptr; if (KindInt->getZExtValue() != ValueKind) - return false; + return nullptr; + + return MD; +} +static bool getValueProfDataFromInstImpl(const MDNode *const MD, + const uint32_t MaxNumDataWant, + InstrProfValueData ValueData[], + uint32_t &ActualNumValueData, + uint64_t &TotalC, bool GetNoICPValue) { + const unsigned NOps = MD->getNumOperands(); // Get total count ConstantInt *TotalCInt = mdconst::dyn_extract(MD->getOperand(2)); if (!TotalCInt) return false; TotalC = TotalCInt->getZExtValue(); - ActualNumValueData = 0; for (unsigned I = 3; I < NOps; I += 2) { - if (ActualNumValueData >= MaxNumValueData) + if (ActualNumValueData >= MaxNumDataWant) break; ConstantInt *Value = mdconst::dyn_extract(MD->getOperand(I)); ConstantInt *Count = @@ -1327,6 +1325,36 @@ bool getValueProfDataFromInst(const Instruction &Inst, return true; } +std::unique_ptr +getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, uint32_t &ActualNumValueData, + uint64_t &TotalC, bool GetNoICPValue) { + MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind); + if (!MD) + return nullptr; + auto ValueDataArray = std::make_unique(MaxNumValueData); + if (!getValueProfDataFromInstImpl(MD, MaxNumValueData, ValueDataArray.get(), + ActualNumValueData, TotalC, GetNoICPValue)) + return nullptr; + return ValueDataArray; +} + +// FIXME: Migrate existing callers to the function above that returns an +// array. +bool getValueProfDataFromInst(const Instruction &Inst, + InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, + InstrProfValueData ValueData[], + uint32_t &ActualNumValueData, uint64_t &TotalC, + bool GetNoICPValue) { + MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind); + if (!MD) + return false; + return getValueProfDataFromInstImpl(MD, MaxNumValueData, ValueData, + ActualNumValueData, TotalC, + GetNoICPValue); +} + MDNode *getPGOFuncNameMetadata(const Function &F) { return F.getMetadata(getPGOFuncNameMetadataName()); } diff --git a/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll index 78b175caca85..ba3ce9a75ee8 100644 --- a/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll +++ b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll @@ -1,20 +1,31 @@ -; RUN: opt -module-summary %s -o %t.o +; Promote at most one function and annotate at most one vtable. +; As a result, only one value (of each relevant kind) shows up in the function +; summary. + +; RUN: opt -module-summary -icp-max-num-vtables=1 -icp-max-prom=1 %s -o %t.o ; RUN: llvm-bcanalyzer -dump %t.o | FileCheck %s ; RUN: llvm-dis -o - %t.o | FileCheck %s --check-prefix=DIS - +; Round trip it through llvm-as +; RUN: llvm-dis -o - %t.o | llvm-as -o - | llvm-dis -o - | FileCheck %s --check-prefix=DIS ; CHECK: ; CHECK-NEXT: +; The `VALUE_GUID` below represents the "_ZTV4Base" referenced by the instruction +; that loads vtable pointers. +; CHECK-NEXT: ; The `VALUE_GUID` below represents the "_ZN4Base4funcEv" referenced by the ; indirect call instruction. -; CHECK-NEXT: +; CHECK-NEXT: +; NOTE vtables and functions from Derived class is dropped because +; `-icp-max-num-vtables` and `-icp-max-prom` are both set to one. ; has the format [valueid, flags, instcount, funcflags, ; numrefs, rorefcnt, worefcnt, +; m x valueid, ; n x (valueid, hotness+tailcall)] -; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" @@ -31,7 +42,6 @@ define i32 @_Z4testP4Base(ptr %0) !prof !15 { !llvm.module.flags = !{!1} - !1 = !{i32 1, !"ProfileSummary", !2} !2 = !{!3, !4, !5, !6, !7, !8, !9, !10} !3 = !{!"ProfileFormat", !"InstrProf"} @@ -48,14 +58,17 @@ define i32 @_Z4testP4Base(ptr %0) !prof !15 { !14 = !{i32 999999, i64 1, i32 2} !15 = !{!"function_entry_count", i32 150} -; 1960855528937986108 is the MD5 hash of _ZTV4Base -!16 = !{!"VP", i32 2, i64 1600, i64 1960855528937986108, i64 1600} -; 5459407273543877811 is the MD5 hash of _ZN4Base4funcEv -!17 = !{!"VP", i32 0, i64 1600, i64 5459407273543877811, i64 1600} +; 1960855528937986108 is the MD5 hash of _ZTV4Base, and +; 13870436605473471591 is the MD5 hash of _ZTV7Derived +!16 = !{!"VP", i32 2, i64 150, i64 1960855528937986108, i64 100, i64 13870436605473471591, i64 50} +; 5459407273543877811 is the MD5 hash of _ZN4Base4funcEv, and +; 6174874150489409711 is the MD5 hash of _ZN7Derived4funcEv +!17 = !{!"VP", i32 0, i64 150, i64 5459407273543877811, i64 100, i64 6174874150489409711, i64 50} ; ModuleSummaryIndex stores map in std::map; so ; global value summares are printed out in the order that gv's guid increases. ; DIS: ^0 = module: (path: "{{.*}}", hash: (0, 0, 0, 0, 0)) -; DIS: ^1 = gv: (guid: 5459407273543877811) -; DIS: ^2 = gv: (name: "_Z4testP4Base", summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 4, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 1, mustBeUnreachable: 0), calls: ((callee: ^1, hotness: hot))))) ; guid = 15857150948103218965 -; DIS: ^3 = blockcount: 0 +; DIS: ^1 = gv: (guid: 1960855528937986108) +; DIS: ^2 = gv: (guid: 5459407273543877811) +; DIS: ^3 = gv: (name: "_Z4testP4Base", summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 4, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 1, mustBeUnreachable: 0), calls: ((callee: ^2, hotness: hot)), refs: (readonly ^1)))) ; guid = 15857150948103218965 +; DIS: ^4 = blockcount: 0 -- GitLab From 649f9603a2da82a32830ce1dc7ce5825d3766a1d Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 1 Apr 2024 15:17:24 -0700 Subject: [PATCH 0146/1690] [workflows] issue-write: Avoid race condition when PR branch is deleted (#87118) Fixes #87102 . --- .github/workflows/issue-write.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml index f5b84fec17a7..4a564a5076ba 100644 --- a/.github/workflows/issue-write.yml +++ b/.github/workflows/issue-write.yml @@ -77,6 +77,15 @@ jobs: } const gql_result = await github.graphql(gql_query, gql_variables); console.log(gql_result); + // If the branch for the PR was deleted before this job has a chance + // to run, then the ref will be null. This can happen if someone: + // 1. Rebase the PR, which triggers some workflow. + // 2. Immediately merges the PR and deletes the branch. + // 3. The workflow finishes and triggers this job. + if (!gql_result.repository.ref) { + console.log("Ref has been deleted"); + return; + } console.log(gql_result.repository.ref.associatedPullRequests.nodes); var pr_number = 0; -- GitLab From f2a87b07e7fe1892a11ee9424d22dbaec5de5b5b Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 1 Apr 2024 17:26:20 -0500 Subject: [PATCH 0147/1690] [OpenMP] Use loaded offloading toolchains to add libraries (#87108) Summary: We want to pass these GPU libraries by default if a certain offloading toolchain is loaded for OpenMP. Previously I parsed this from the arguments because it's only available in the compilation. This doesn't really work for `native` and it's extra effort, so this patch just passes in the `Compilation` as an extr argument and uses that. Tests should be unaffected. --- clang/lib/Driver/ToolChains/CommonArgs.cpp | 58 ++++++++-------------- clang/lib/Driver/ToolChains/CommonArgs.h | 4 +- clang/lib/Driver/ToolChains/Darwin.cpp | 2 +- clang/lib/Driver/ToolChains/DragonFly.cpp | 2 +- clang/lib/Driver/ToolChains/FreeBSD.cpp | 2 +- clang/lib/Driver/ToolChains/Gnu.cpp | 2 +- clang/lib/Driver/ToolChains/Haiku.cpp | 2 +- clang/lib/Driver/ToolChains/NetBSD.cpp | 2 +- clang/lib/Driver/ToolChains/OpenBSD.cpp | 2 +- clang/lib/Driver/ToolChains/Solaris.cpp | 2 +- 10 files changed, 32 insertions(+), 46 deletions(-) diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index ace4fb99581e..62a53b85ce09 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1075,14 +1075,14 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, /// Adds the '-lcgpu' and '-lmgpu' libraries to the compilation to include the /// LLVM C library for GPUs. -static void addOpenMPDeviceLibC(const ToolChain &TC, const ArgList &Args, +static void addOpenMPDeviceLibC(const Compilation &C, const ArgList &Args, ArgStringList &CmdArgs) { if (Args.hasArg(options::OPT_nogpulib) || Args.hasArg(options::OPT_nolibc)) return; // Check the resource directory for the LLVM libc GPU declarations. If it's // found we can assume that LLVM was built with support for the GPU libc. - SmallString<256> LibCDecls(TC.getDriver().ResourceDir); + SmallString<256> LibCDecls(C.getDriver().ResourceDir); llvm::sys::path::append(LibCDecls, "include", "llvm_libc_wrappers", "llvm-libc-decls"); bool HasLibC = llvm::sys::fs::exists(LibCDecls) && @@ -1090,38 +1090,23 @@ static void addOpenMPDeviceLibC(const ToolChain &TC, const ArgList &Args, if (!Args.hasFlag(options::OPT_gpulibc, options::OPT_nogpulibc, HasLibC)) return; - // We don't have access to the offloading toolchains here, so determine from - // the arguments if we have any active NVPTX or AMDGPU toolchains. - llvm::DenseSet Libraries; - if (const Arg *Targets = Args.getLastArg(options::OPT_fopenmp_targets_EQ)) { - if (llvm::any_of(Targets->getValues(), - [](auto S) { return llvm::Triple(S).isAMDGPU(); })) { - Libraries.insert("-lcgpu-amdgpu"); - Libraries.insert("-lmgpu-amdgpu"); - } - if (llvm::any_of(Targets->getValues(), - [](auto S) { return llvm::Triple(S).isNVPTX(); })) { - Libraries.insert("-lcgpu-nvptx"); - Libraries.insert("-lmgpu-nvptx"); - } - } + SmallVector ToolChains; + auto TCRange = C.getOffloadToolChains(Action::OFK_OpenMP); + for (auto TI = TCRange.first, TE = TCRange.second; TI != TE; ++TI) + ToolChains.push_back(TI->second); - for (StringRef Arch : Args.getAllArgValues(options::OPT_offload_arch_EQ)) { - if (llvm::any_of(llvm::split(Arch, ","), [](StringRef Str) { - return IsAMDGpuArch(StringToCudaArch(Str)); - })) { - Libraries.insert("-lcgpu-amdgpu"); - Libraries.insert("-lmgpu-amdgpu"); - } - if (llvm::any_of(llvm::split(Arch, ","), [](StringRef Str) { - return IsNVIDIAGpuArch(StringToCudaArch(Str)); - })) { - Libraries.insert("-lcgpu-nvptx"); - Libraries.insert("-lmgpu-nvptx"); - } + if (llvm::any_of(ToolChains, [](const ToolChain *TC) { + return TC->getTriple().isAMDGPU(); + })) { + CmdArgs.push_back("-lcgpu-amdgpu"); + CmdArgs.push_back("-lmgpu-amdgpu"); + } + if (llvm::any_of(ToolChains, [](const ToolChain *TC) { + return TC->getTriple().isNVPTX(); + })) { + CmdArgs.push_back("-lcgpu-nvptx"); + CmdArgs.push_back("-lmgpu-nvptx"); } - - llvm::append_range(CmdArgs, Libraries); } void tools::addOpenMPRuntimeLibraryPath(const ToolChain &TC, @@ -1153,9 +1138,10 @@ void tools::addArchSpecificRPath(const ToolChain &TC, const ArgList &Args, } } -bool tools::addOpenMPRuntime(ArgStringList &CmdArgs, const ToolChain &TC, - const ArgList &Args, bool ForceStaticHostRuntime, - bool IsOffloadingHost, bool GompNeedsRT) { +bool tools::addOpenMPRuntime(const Compilation &C, ArgStringList &CmdArgs, + const ToolChain &TC, const ArgList &Args, + bool ForceStaticHostRuntime, bool IsOffloadingHost, + bool GompNeedsRT) { if (!Args.hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ, options::OPT_fno_openmp, false)) return false; @@ -1196,7 +1182,7 @@ bool tools::addOpenMPRuntime(ArgStringList &CmdArgs, const ToolChain &TC, CmdArgs.push_back("-lomptarget.devicertl"); if (IsOffloadingHost) - addOpenMPDeviceLibC(TC, Args, CmdArgs); + addOpenMPDeviceLibC(C, Args, CmdArgs); addArchSpecificRPath(TC, Args, CmdArgs); addOpenMPRuntimeLibraryPath(TC, Args, CmdArgs); diff --git a/clang/lib/Driver/ToolChains/CommonArgs.h b/clang/lib/Driver/ToolChains/CommonArgs.h index bb37be4bd6ea..5581905db311 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.h +++ b/clang/lib/Driver/ToolChains/CommonArgs.h @@ -111,8 +111,8 @@ void addOpenMPRuntimeLibraryPath(const ToolChain &TC, const llvm::opt::ArgList &Args, llvm::opt::ArgStringList &CmdArgs); /// Returns true, if an OpenMP runtime has been added. -bool addOpenMPRuntime(llvm::opt::ArgStringList &CmdArgs, const ToolChain &TC, - const llvm::opt::ArgList &Args, +bool addOpenMPRuntime(const Compilation &C, llvm::opt::ArgStringList &CmdArgs, + const ToolChain &TC, const llvm::opt::ArgList &Args, bool ForceStaticHostRuntime = false, bool IsOffloadingHost = false, bool GompNeedsRT = false); diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp index c7682c7f1d33..caf6c4a444fd 100644 --- a/clang/lib/Driver/ToolChains/Darwin.cpp +++ b/clang/lib/Driver/ToolChains/Darwin.cpp @@ -686,7 +686,7 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA, } if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) - addOpenMPRuntime(CmdArgs, getToolChain(), Args); + addOpenMPRuntime(C, CmdArgs, getToolChain(), Args); if (isObjCRuntimeLinked(Args) && !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) { diff --git a/clang/lib/Driver/ToolChains/DragonFly.cpp b/clang/lib/Driver/ToolChains/DragonFly.cpp index b59a172bd6ae..1dbc46763c11 100644 --- a/clang/lib/Driver/ToolChains/DragonFly.cpp +++ b/clang/lib/Driver/ToolChains/DragonFly.cpp @@ -136,7 +136,7 @@ void dragonfly::Linker::ConstructJob(Compilation &C, const JobAction &JA, // Use the static OpenMP runtime with -static-openmp bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) && !Static; - addOpenMPRuntime(CmdArgs, ToolChain, Args, StaticOpenMP); + addOpenMPRuntime(C, CmdArgs, ToolChain, Args, StaticOpenMP); if (D.CCCIsCXX()) { if (ToolChain.ShouldLinkCXXStdlib(Args)) diff --git a/clang/lib/Driver/ToolChains/FreeBSD.cpp b/clang/lib/Driver/ToolChains/FreeBSD.cpp index c5757ddebb0f..a8ee6540001e 100644 --- a/clang/lib/Driver/ToolChains/FreeBSD.cpp +++ b/clang/lib/Driver/ToolChains/FreeBSD.cpp @@ -295,7 +295,7 @@ void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, // Use the static OpenMP runtime with -static-openmp bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) && !Args.hasArg(options::OPT_static); - addOpenMPRuntime(CmdArgs, ToolChain, Args, StaticOpenMP); + addOpenMPRuntime(C, CmdArgs, ToolChain, Args, StaticOpenMP); if (D.CCCIsCXX()) { if (ToolChain.ShouldLinkCXXStdlib(Args)) diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp index a9c9d2475809..dedbfac6cb25 100644 --- a/clang/lib/Driver/ToolChains/Gnu.cpp +++ b/clang/lib/Driver/ToolChains/Gnu.cpp @@ -598,7 +598,7 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA, // FIXME: Only pass GompNeedsRT = true for platforms with libgomp that // require librt. Most modern Linux platforms do, but some may not. - if (addOpenMPRuntime(CmdArgs, ToolChain, Args, StaticOpenMP, + if (addOpenMPRuntime(C, CmdArgs, ToolChain, Args, StaticOpenMP, JA.isHostOffloading(Action::OFK_OpenMP), /* GompNeedsRT= */ true)) // OpenMP runtimes implies pthreads when using the GNU toolchain. diff --git a/clang/lib/Driver/ToolChains/Haiku.cpp b/clang/lib/Driver/ToolChains/Haiku.cpp index 30464e2229e6..346652a7e4bd 100644 --- a/clang/lib/Driver/ToolChains/Haiku.cpp +++ b/clang/lib/Driver/ToolChains/Haiku.cpp @@ -107,7 +107,7 @@ void haiku::Linker::ConstructJob(Compilation &C, const JobAction &JA, options::OPT_r)) { // Use the static OpenMP runtime with -static-openmp bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) && !Static; - addOpenMPRuntime(CmdArgs, ToolChain, Args, StaticOpenMP); + addOpenMPRuntime(C, CmdArgs, ToolChain, Args, StaticOpenMP); if (D.CCCIsCXX() && ToolChain.ShouldLinkCXXStdlib(Args)) ToolChain.AddCXXStdlibLibArgs(Args, CmdArgs); diff --git a/clang/lib/Driver/ToolChains/NetBSD.cpp b/clang/lib/Driver/ToolChains/NetBSD.cpp index 0eec8fddabd5..d54f22882949 100644 --- a/clang/lib/Driver/ToolChains/NetBSD.cpp +++ b/clang/lib/Driver/ToolChains/NetBSD.cpp @@ -311,7 +311,7 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, options::OPT_r)) { // Use the static OpenMP runtime with -static-openmp bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) && !Static; - addOpenMPRuntime(CmdArgs, ToolChain, Args, StaticOpenMP); + addOpenMPRuntime(C, CmdArgs, ToolChain, Args, StaticOpenMP); if (D.CCCIsCXX()) { if (ToolChain.ShouldLinkCXXStdlib(Args)) diff --git a/clang/lib/Driver/ToolChains/OpenBSD.cpp b/clang/lib/Driver/ToolChains/OpenBSD.cpp index 6da6728585df..e20d9fb1cfc4 100644 --- a/clang/lib/Driver/ToolChains/OpenBSD.cpp +++ b/clang/lib/Driver/ToolChains/OpenBSD.cpp @@ -221,7 +221,7 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, options::OPT_r)) { // Use the static OpenMP runtime with -static-openmp bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) && !Static; - addOpenMPRuntime(CmdArgs, ToolChain, Args, StaticOpenMP); + addOpenMPRuntime(C, CmdArgs, ToolChain, Args, StaticOpenMP); if (D.CCCIsCXX()) { if (ToolChain.ShouldLinkCXXStdlib(Args)) diff --git a/clang/lib/Driver/ToolChains/Solaris.cpp b/clang/lib/Driver/ToolChains/Solaris.cpp index 5d7f0ae2a392..7126e018ca5b 100644 --- a/clang/lib/Driver/ToolChains/Solaris.cpp +++ b/clang/lib/Driver/ToolChains/Solaris.cpp @@ -211,7 +211,7 @@ void solaris::Linker::ConstructJob(Compilation &C, const JobAction &JA, // Use the static OpenMP runtime with -static-openmp bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) && !Args.hasArg(options::OPT_static); - addOpenMPRuntime(CmdArgs, ToolChain, Args, StaticOpenMP); + addOpenMPRuntime(C, CmdArgs, ToolChain, Args, StaticOpenMP); if (D.CCCIsCXX()) { if (ToolChain.ShouldLinkCXXStdlib(Args)) -- GitLab From 9df19ce40281551bd348b262a131085cf98dadf5 Mon Sep 17 00:00:00 2001 From: David Blaikie Date: Mon, 1 Apr 2024 23:07:01 +0000 Subject: [PATCH 0148/1690] Add uncovered enums in switches caused by 9434c083475e42f47383f3067fe2a155db5c6a30 These are probably actually unreachable - perhaps an lldb developer would be interested in rephrasing this change to move the new cases into some unreachable/unsupported bucket, rather than my half-hearted guess at what the desired behavior would be (completely untested, because they're probably untestable/unreachable - maybe debugging from modules?) --- lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp index ebcc3bc99a80..4a1c8d576552 100644 --- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp +++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp @@ -4097,6 +4097,8 @@ TypeSystemClang::GetTypeClass(lldb::opaque_compiler_type_t type) { return lldb::eTypeClassArray; case clang::Type::DependentSizedArray: return lldb::eTypeClassArray; + case clang::Type::ArrayParameter: + return lldb::eTypeClassArray; case clang::Type::DependentSizedExtVector: return lldb::eTypeClassVector; case clang::Type::DependentVector: @@ -4776,6 +4778,7 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type, case clang::Type::IncompleteArray: case clang::Type::VariableArray: + case clang::Type::ArrayParameter: break; case clang::Type::ConstantArray: @@ -5109,6 +5112,7 @@ lldb::Format TypeSystemClang::GetFormat(lldb::opaque_compiler_type_t type) { case clang::Type::IncompleteArray: case clang::Type::VariableArray: + case clang::Type::ArrayParameter: break; case clang::Type::ConstantArray: -- GitLab From 1079fc4f543c42bb09a33d2d79d90edd9c0bac91 Mon Sep 17 00:00:00 2001 From: Ivan Butygin Date: Tue, 2 Apr 2024 02:43:04 +0300 Subject: [PATCH 0149/1690] [mlir][pass] Add `errorHandler` param to `Pass::initializeOptions` (#87289) There is no good way to report detailed errors from inside `Pass::initializeOptions` function as context may not be available at this point and writing directly to `llvm::errs()` is not composable. See https://github.com/llvm/llvm-project/pull/87166#discussion_r1546426763 * Add error handler callback to `Pass::initializeOptions` * Update `PassOptions::parseFromString` to support custom error stream instead of using `llvm::errs()` directly. * Update default `Pass::initializeOptions` implementation to propagate error string from `parseFromString` to new error handler. * Update `MapMemRefStorageClassPass` to report error details using new API. --- mlir/include/mlir/Pass/Pass.h | 4 +++- mlir/include/mlir/Pass/PassOptions.h | 3 ++- .../MemRefToSPIRV/MapMemRefStorageClassPass.cpp | 8 +++++--- mlir/lib/Pass/Pass.cpp | 12 ++++++++++-- mlir/lib/Pass/PassRegistry.cpp | 7 ++++--- mlir/lib/Transforms/InlinerPass.cpp | 10 +++++++--- .../Dialect/Transform/test-pass-application.mlir | 1 + 7 files changed, 32 insertions(+), 13 deletions(-) diff --git a/mlir/include/mlir/Pass/Pass.h b/mlir/include/mlir/Pass/Pass.h index 070e0cad3878..0f50f3064f17 100644 --- a/mlir/include/mlir/Pass/Pass.h +++ b/mlir/include/mlir/Pass/Pass.h @@ -114,7 +114,9 @@ public: /// Derived classes may override this method to hook into the point at which /// options are initialized, but should generally always invoke this base /// class variant. - virtual LogicalResult initializeOptions(StringRef options); + virtual LogicalResult + initializeOptions(StringRef options, + function_ref errorHandler); /// Prints out the pass in the textual representation of pipelines. If this is /// an adaptor pass, print its pass managers. diff --git a/mlir/include/mlir/Pass/PassOptions.h b/mlir/include/mlir/Pass/PassOptions.h index 6717a3585d12..3a5e3224133e 100644 --- a/mlir/include/mlir/Pass/PassOptions.h +++ b/mlir/include/mlir/Pass/PassOptions.h @@ -293,7 +293,8 @@ public: /// Parse options out as key=value pairs that can then be handed off to the /// `llvm::cl` command line passing infrastructure. Everything is space /// separated. - LogicalResult parseFromString(StringRef options); + LogicalResult parseFromString(StringRef options, + raw_ostream &errorStream = llvm::errs()); /// Print the options held by this struct in a form that can be parsed via /// 'parseFromString'. diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp index 76dab8ee4ac3..4cbc3dfdae22 100644 --- a/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp +++ b/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp @@ -272,14 +272,16 @@ public: const spirv::MemorySpaceToStorageClassMap &memorySpaceMap) : memorySpaceMap(memorySpaceMap) {} - LogicalResult initializeOptions(StringRef options) override { - if (failed(Pass::initializeOptions(options))) + LogicalResult initializeOptions( + StringRef options, + function_ref errorHandler) override { + if (failed(Pass::initializeOptions(options, errorHandler))) return failure(); if (clientAPI == "opencl") memorySpaceMap = spirv::mapMemorySpaceToOpenCLStorageClass; else if (clientAPI != "vulkan") - return failure(); + return errorHandler(llvm::Twine("Invalid clienAPI: ") + clientAPI); return success(); } diff --git a/mlir/lib/Pass/Pass.cpp b/mlir/lib/Pass/Pass.cpp index 3fb05e538666..57a6c20141d2 100644 --- a/mlir/lib/Pass/Pass.cpp +++ b/mlir/lib/Pass/Pass.cpp @@ -60,8 +60,16 @@ Operation *PassExecutionAction::getOp() const { void Pass::anchor() {} /// Attempt to initialize the options of this pass from the given string. -LogicalResult Pass::initializeOptions(StringRef options) { - return passOptions.parseFromString(options); +LogicalResult Pass::initializeOptions( + StringRef options, + function_ref errorHandler) { + std::string errStr; + llvm::raw_string_ostream os(errStr); + if (failed(passOptions.parseFromString(options, os))) { + os.flush(); + return errorHandler(errStr); + } + return success(); } /// Copy the option values from 'other', which is another instance of this diff --git a/mlir/lib/Pass/PassRegistry.cpp b/mlir/lib/Pass/PassRegistry.cpp index b0c314369190..f8149673a409 100644 --- a/mlir/lib/Pass/PassRegistry.cpp +++ b/mlir/lib/Pass/PassRegistry.cpp @@ -40,7 +40,7 @@ buildDefaultRegistryFn(const PassAllocatorFunction &allocator) { return [=](OpPassManager &pm, StringRef options, function_ref errorHandler) { std::unique_ptr pass = allocator(); - LogicalResult result = pass->initializeOptions(options); + LogicalResult result = pass->initializeOptions(options, errorHandler); std::optional pmOpName = pm.getOpName(); std::optional passOpName = pass->getOpName(); @@ -280,7 +280,8 @@ parseNextArg(StringRef options) { llvm_unreachable("unexpected control flow in pass option parsing"); } -LogicalResult detail::PassOptions::parseFromString(StringRef options) { +LogicalResult detail::PassOptions::parseFromString(StringRef options, + raw_ostream &errorStream) { // NOTE: `options` is modified in place to always refer to the unprocessed // part of the string. while (!options.empty()) { @@ -291,7 +292,7 @@ LogicalResult detail::PassOptions::parseFromString(StringRef options) { auto it = OptionsMap.find(key); if (it == OptionsMap.end()) { - llvm::errs() << ": no such option " << key << "\n"; + errorStream << ": no such option " << key << "\n"; return failure(); } if (llvm::cl::ProvidePositionalOption(it->second, value, 0)) diff --git a/mlir/lib/Transforms/InlinerPass.cpp b/mlir/lib/Transforms/InlinerPass.cpp index 9a7d5403a95d..43ca5cac8b76 100644 --- a/mlir/lib/Transforms/InlinerPass.cpp +++ b/mlir/lib/Transforms/InlinerPass.cpp @@ -64,7 +64,9 @@ private: /// Derived classes may override this method to hook into the point at which /// options are initialized, but should generally always invoke this base /// class variant. - LogicalResult initializeOptions(StringRef options) override; + LogicalResult initializeOptions( + StringRef options, + function_ref errorHandler) override; /// Inliner configuration parameters created from the pass options. InlinerConfig config; @@ -153,8 +155,10 @@ void InlinerPass::runOnOperation() { return; } -LogicalResult InlinerPass::initializeOptions(StringRef options) { - if (failed(Pass::initializeOptions(options))) +LogicalResult InlinerPass::initializeOptions( + StringRef options, + function_ref errorHandler) { + if (failed(Pass::initializeOptions(options, errorHandler))) return failure(); // Initialize the pipeline builder for operations without the dedicated diff --git a/mlir/test/Dialect/Transform/test-pass-application.mlir b/mlir/test/Dialect/Transform/test-pass-application.mlir index 7cb5387b937d..460ac3947f5c 100644 --- a/mlir/test/Dialect/Transform/test-pass-application.mlir +++ b/mlir/test/Dialect/Transform/test-pass-application.mlir @@ -78,6 +78,7 @@ module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg1: !transform.any_op) { %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op // expected-error @below {{failed to add pass or pass pipeline to pipeline: canonicalize}} + // expected-error @below {{: no such option invalid-option}} transform.apply_registered_pass "canonicalize" to %1 {options = "invalid-option=1"} : (!transform.any_op) -> !transform.any_op transform.yield } -- GitLab From 6d0174e70641b1ea172ffed07c43604ef15e28ae Mon Sep 17 00:00:00 2001 From: Stephen Neuendorffer Date: Mon, 1 Apr 2024 17:04:29 -0700 Subject: [PATCH 0150/1690] [libc] allow libc-hdrgen to work on windows files (#87292) The code does some (overly simple?) checks on file syntax. These checks assume unix line endings and fail on windows. This commit updates the code to strip extra whitespace, making the checks more robust, particularly in the presence of windows line endings. Fixes #86023 --- libc/utils/HdrGen/Generator.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libc/utils/HdrGen/Generator.cpp b/libc/utils/HdrGen/Generator.cpp index 3bcf005adda7..d926d5d9ac3c 100644 --- a/libc/utils/HdrGen/Generator.cpp +++ b/libc/utils/HdrGen/Generator.cpp @@ -84,11 +84,19 @@ void Generator::generate(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) { Line = Line.drop_front(CommandPrefixSize); P = Line.split("("); + // It's possible that we have windows line endings, so strip off the extra + // CR. + P.second = P.second.trim(); if (P.second.empty() || P.second[P.second.size() - 1] != ')') { SrcMgr.PrintMessage(llvm::SMLoc::getFromPointer(P.second.data()), llvm::SourceMgr::DK_Error, "Command argument list should begin with '(' " "and end with ')'."); + SrcMgr.PrintMessage(llvm::SMLoc::getFromPointer(P.second.data()), + llvm::SourceMgr::DK_Error, P.second.data()); + SrcMgr.PrintMessage(llvm::SMLoc::getFromPointer(P.second.data()), + llvm::SourceMgr::DK_Error, + std::to_string(P.second.size())); std::exit(1); } llvm::StringRef CommandName = P.first; -- GitLab From dd5797505ebc2dbfdd58927c4f0a11a1256696eb Mon Sep 17 00:00:00 2001 From: Abhinav Gunjal Date: Mon, 1 Apr 2024 17:36:09 -0700 Subject: [PATCH 0151/1690] lit_test : check if there is already a deps key in kwargs (#87290) This change checks if there is already a `deps` key in `kwargs` and concatenate it to avoid multiple values for `deps` key argument. background: https://github.com/llvm/llvm-project/pull/87022 recently added explicit `deps` to the lit_test. This is causing StableHLO bazel build failures at https://github.com/openxla/stablehlo/actions/runs/8511888283/job/23312383380?pr=2147 Tested: local build run is successful --- utils/bazel/llvm-project-overlay/llvm/lit_test.bzl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/bazel/llvm-project-overlay/llvm/lit_test.bzl b/utils/bazel/llvm-project-overlay/llvm/lit_test.bzl index f754a9fc7d5e..af7ae560768d 100644 --- a/utils/bazel/llvm-project-overlay/llvm/lit_test.bzl +++ b/utils/bazel/llvm-project-overlay/llvm/lit_test.bzl @@ -10,6 +10,7 @@ def lit_test( srcs, args = None, data = None, + deps = None, **kwargs): """Runs a single test file with LLVM's lit tool. @@ -27,6 +28,7 @@ def lit_test( args = args or [] data = data or [] + deps = deps or [] native.py_test( name = name, @@ -35,7 +37,7 @@ def lit_test( args = args + ["-v"] + ["$(execpath %s)" % src for src in srcs], data = data + srcs, legacy_create_init = False, - deps = [Label("//llvm:lit")], + deps = deps + [Label("//llvm:lit")], **kwargs ) -- GitLab From 9dbd364589883ae3343a291077804c564d4b3de5 Mon Sep 17 00:00:00 2001 From: Ben Shi <2283975856@qq.com> Date: Tue, 2 Apr 2024 08:38:02 +0800 Subject: [PATCH 0152/1690] [AVR][NFC] Improve format of target description files (#87212) --- llvm/lib/Target/AVR/AVRInstrInfo.td | 320 +++++++--------------------- 1 file changed, 75 insertions(+), 245 deletions(-) diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.td b/llvm/lib/Target/AVR/AVRInstrInfo.td index fe0d3b6c8189..38ebfab64c61 100644 --- a/llvm/lib/Target/AVR/AVRInstrInfo.td +++ b/llvm/lib/Target/AVR/AVRInstrInfo.td @@ -343,13 +343,9 @@ def AVR_COND_PL : PatLeaf<(i8 7)>; // Pessimistically assume ADJCALLSTACKDOWN / ADJCALLSTACKUP will become // sub / add which can clobber SREG. let Defs = [SP, SREG], Uses = [SP] in { - def ADJCALLSTACKDOWN : Pseudo<(outs), - (ins i16imm - : $amt, i16imm - : $amt2), - "#ADJCALLSTACKDOWN", [(AVRcallseq_start timm - : $amt, timm - : $amt2)]>; + def ADJCALLSTACKDOWN : Pseudo<(outs), (ins i16imm:$amt, i16imm:$amt2), + "#ADJCALLSTACKDOWN", + [(AVRcallseq_start timm:$amt, timm:$amt2)]>; // R31R30 is used to update SP. It is normally free because it is a // call-clobbered register but it is necessary to set it as a def as the @@ -357,13 +353,8 @@ let Defs = [SP, SREG], Uses = [SP] in { // seems). hasSideEffects needs to be set to true so this instruction isn't // considered dead. let Defs = [R31R30], hasSideEffects = 1 in def ADJCALLSTACKUP - : Pseudo<(outs), - (ins i16imm - : $amt1, i16imm - : $amt2), - "#ADJCALLSTACKUP", [(AVRcallseq_end timm - : $amt1, timm - : $amt2)]>; + : Pseudo<(outs), (ins i16imm:$amt1, i16imm:$amt2), + "#ADJCALLSTACKUP", [(AVRcallseq_end timm:$amt1, timm:$amt2)]>; } //===----------------------------------------------------------------------===// @@ -372,19 +363,9 @@ let Defs = [SP, SREG], Uses = [SP] in { let isCommutable = 1, Constraints = "$src = $rd", Defs = [SREG] in { // ADD Rd, Rr // Adds two 8-bit registers. - def ADDRdRr - : FRdRr<0b0000, 0b11, - (outs GPR8 - : $rd), - (ins GPR8 - : $src, GPR8 - : $rr), - "add\t$rd, $rr", - [(set i8 - : $rd, (add i8 - : $src, i8 - : $rr)), - (implicit SREG)]>; + def ADDRdRr : FRdRr<0b0000, 0b11, (outs GPR8:$rd),(ins GPR8:$src, GPR8:$rr), + "add\t$rd, $rr", + [(set i8:$rd, (add i8:$src, i8:$rr)), (implicit SREG)]>; // ADDW Rd+1:Rd, Rr+1:Rr // Pseudo instruction to add four 8-bit registers as two 16-bit values. @@ -392,34 +373,17 @@ let isCommutable = 1, Constraints = "$src = $rd", Defs = [SREG] in { // Expands to: // add Rd, Rr // adc Rd+1, Rr+1 - def ADDWRdRr - : Pseudo<(outs DREGS - : $rd), - (ins DREGS - : $src, DREGS - : $rr), - "addw\t$rd, $rr", - [(set i16 - : $rd, (add i16 - : $src, i16 - : $rr)), - (implicit SREG)]>; + def ADDWRdRr : Pseudo<(outs DREGS:$rd), (ins DREGS:$src, DREGS:$rr), + "addw\t$rd, $rr", + [(set i16:$rd, (add i16:$src, i16:$rr)), + (implicit SREG)]>; // ADC Rd, Rr // Adds two 8-bit registers with carry. - let Uses = [SREG] in def ADCRdRr - : FRdRr<0b0001, 0b11, - (outs GPR8 - : $rd), - (ins GPR8 - : $src, GPR8 - : $rr), - "adc\t$rd, $rr", - [(set i8 - : $rd, (adde i8 - : $src, i8 - : $rr)), - (implicit SREG)]>; + let Uses = [SREG] in + def ADCRdRr : FRdRr<0b0001, 0b11, (outs GPR8:$rd), (ins GPR8:$src, GPR8:$rr), + "adc\t$rd, $rr", + [(set i8:$rd, (adde i8:$src, i8:$rr)), (implicit SREG)]>; // ADCW Rd+1:Rd, Rr+1:Rr // Pseudo instruction to add four 8-bit registers as two 16-bit values with @@ -428,56 +392,30 @@ let isCommutable = 1, Constraints = "$src = $rd", Defs = [SREG] in { // Expands to: // adc Rd, Rr // adc Rd+1, Rr+1 - let Uses = [SREG] in def ADCWRdRr : Pseudo<(outs DREGS - : $rd), - (ins DREGS - : $src, DREGS - : $rr), - "adcw\t$rd, $rr", [ - (set i16 - : $rd, (adde i16 - : $src, i16 - : $rr)), - (implicit SREG) - ]>; + let Uses = [SREG] in + def ADCWRdRr : Pseudo<(outs DREGS:$rd), (ins DREGS:$src, DREGS:$rr), + "adcw\t$rd, $rr", + [(set i16:$rd, (adde i16:$src, i16:$rr)), + (implicit SREG)]>; // AIDW Rd, k // Adds an immediate 6-bit value K to Rd, placing the result in Rd. - def ADIWRdK - : FWRdK<0b0, - (outs IWREGS - : $rd), - (ins IWREGS - : $src, imm_arith6 - : $k), - "adiw\t$rd, $k", - [(set i16 - : $rd, (add i16 - : $src, uimm6 - : $k)), - (implicit SREG)]>, - Requires<[HasADDSUBIW]>; + def ADIWRdK : FWRdK<0b0, (outs IWREGS:$rd), (ins IWREGS :$src, imm_arith6:$k), + "adiw\t$rd, $k", + [(set i16:$rd, (add i16:$src, uimm6:$k)), + (implicit SREG)]>, + Requires<[HasADDSUBIW]>; } //===----------------------------------------------------------------------===// // Subtraction //===----------------------------------------------------------------------===// -let Constraints = "$src = $rd", Defs = [SREG] in { +let Constraints = "$rs = $rd", Defs = [SREG] in { // SUB Rd, Rr // Subtracts the 8-bit value of Rr from Rd and places the value in Rd. - def SUBRdRr - : FRdRr<0b0001, 0b10, - (outs GPR8 - : $rd), - (ins GPR8 - : $src, GPR8 - : $rr), - "sub\t$rd, $rr", - [(set i8 - : $rd, (sub i8 - : $src, i8 - : $rr)), - (implicit SREG)]>; + def SUBRdRr : FRdRr<0b0001, 0b10, (outs GPR8:$rd), (ins GPR8:$rs, GPR8:$rr), + "sub\t$rd, $rr", + [(set i8:$rd, (sub i8:$rs, i8:$rr)), (implicit SREG)]>; // SUBW Rd+1:Rd, Rr+1:Rr // Subtracts two 16-bit values and places the result into Rd. @@ -485,129 +423,58 @@ let Constraints = "$src = $rd", Defs = [SREG] in { // Expands to: // sub Rd, Rr // sbc Rd+1, Rr+1 - def SUBWRdRr - : Pseudo<(outs DREGS - : $rd), - (ins DREGS - : $src, DREGS - : $rr), - "subw\t$rd, $rr", - [(set i16 - : $rd, (sub i16 - : $src, i16 - : $rr)), - (implicit SREG)]>; + def SUBWRdRr : Pseudo<(outs DREGS:$rd), (ins DREGS:$rs, DREGS:$rr), + "subw\t$rd, $rr", + [(set i16:$rd, (sub i16:$rs, i16:$rr)), + (implicit SREG)]>; - def SUBIRdK - : FRdK<0b0101, - (outs LD8 - : $rd), - (ins LD8 - : $src, imm_ldi8 - : $k), - "subi\t$rd, $k", - [(set i8 - : $rd, (sub i8 - : $src, imm - : $k)), - (implicit SREG)]>; + def SUBIRdK : FRdK<0b0101, (outs LD8:$rd), (ins LD8:$rs, imm_ldi8:$k), + "subi\t$rd, $k", + [(set i8:$rd, (sub i8:$rs, imm:$k)), (implicit SREG)]>; // SUBIW Rd+1:Rd, K+1:K // // Expands to: // subi Rd, K // sbci Rd+1, K+1 - def SUBIWRdK - : Pseudo<(outs DLDREGS - : $rd), - (ins DLDREGS - : $src, i16imm - : $rr), - "subiw\t$rd, $rr", - [(set i16 - : $rd, (sub i16 - : $src, imm - : $rr)), - (implicit SREG)]>; + def SUBIWRdK : Pseudo<(outs DLDREGS:$rd), (ins DLDREGS:$rs, i16imm:$rr), + "subiw\t$rd, $rr", + [(set i16:$rd, (sub i16:$rs, imm:$rr)), + (implicit SREG)]>; - def SBIWRdK - : FWRdK<0b1, - (outs IWREGS - : $rd), - (ins IWREGS - : $src, imm_arith6 - : $k), - "sbiw\t$rd, $k", - [(set i16 - : $rd, (sub i16 - : $src, uimm6 - : $k)), - (implicit SREG)]>, - Requires<[HasADDSUBIW]>; + def SBIWRdK : FWRdK<0b1, (outs IWREGS:$rd), (ins IWREGS:$rs, imm_arith6:$k), + "sbiw\t$rd, $k", + [(set i16:$rd, (sub i16:$rs, uimm6:$k)), + (implicit SREG)]>, + Requires<[HasADDSUBIW]>; // Subtract with carry operations which must read the carry flag in SREG. let Uses = [SREG] in { - def SBCRdRr - : FRdRr<0b0000, 0b10, - (outs GPR8 - : $rd), - (ins GPR8 - : $src, GPR8 - : $rr), - "sbc\t$rd, $rr", - [(set i8 - : $rd, (sube i8 - : $src, i8 - : $rr)), - (implicit SREG)]>; + def SBCRdRr : FRdRr<0b0000, 0b10, (outs GPR8:$rd), (ins GPR8:$rs, GPR8:$rr), + "sbc\t$rd, $rr", + [(set i8:$rd, (sube i8:$rs, i8:$rr)), (implicit SREG)]>; // SBCW Rd+1:Rd, Rr+1:Rr // // Expands to: // sbc Rd, Rr // sbc Rd+1, Rr+1 - def SBCWRdRr : Pseudo<(outs DREGS - : $rd), - (ins DREGS - : $src, DREGS - : $rr), - "sbcw\t$rd, $rr", [ - (set i16 - : $rd, (sube i16 - : $src, i16 - : $rr)), - (implicit SREG) - ]>; + def SBCWRdRr : Pseudo<(outs DREGS:$rd), (ins DREGS:$rs, DREGS:$rr), + "sbcw\t$rd, $rr", + [(set i16:$rd, (sube i16:$rs, i16:$rr)), + (implicit SREG)]>; - def SBCIRdK - : FRdK<0b0100, - (outs LD8 - : $rd), - (ins LD8 - : $src, imm_ldi8 - : $k), - "sbci\t$rd, $k", - [(set i8 - : $rd, (sube i8 - : $src, imm - : $k)), - (implicit SREG)]>; + def SBCIRdK : FRdK<0b0100, (outs LD8:$rd), (ins LD8:$rs, imm_ldi8:$k), + "sbci\t$rd, $k", + [(set i8:$rd, (sube i8:$rs, imm:$k)), (implicit SREG)]>; // SBCIW Rd+1:Rd, K+1:K // sbci Rd, K // sbci Rd+1, K+1 - def SBCIWRdK : Pseudo<(outs DLDREGS - : $rd), - (ins DLDREGS - : $src, i16imm - : $rr), - "sbciw\t$rd, $rr", [ - (set i16 - : $rd, (sube i16 - : $src, imm - : $rr)), - (implicit SREG) - ]>; + def SBCIWRdK : Pseudo<(outs DLDREGS:$rd), (ins DLDREGS:$rs, i16imm:$rr), + "sbciw\t$rd, $rr", + [(set i16:$rd, (sube i16:$rs, imm:$rr)), + (implicit SREG)]>; } } @@ -615,27 +482,13 @@ let Constraints = "$src = $rd", Defs = [SREG] in { // Increment and Decrement //===----------------------------------------------------------------------===// let Constraints = "$src = $rd", Defs = [SREG] in { - def INCRd - : FRd<0b1001, 0b0100011, - (outs GPR8 - : $rd), - (ins GPR8 - : $src), - "inc\t$rd", [(set i8 - : $rd, (add i8 - : $src, 1)), - (implicit SREG)]>; + def INCRd : FRd<0b1001, 0b0100011, (outs GPR8:$rd), (ins GPR8:$src), + "inc\t$rd", + [(set i8:$rd, (add i8:$src, 1)), (implicit SREG)]>; - def DECRd - : FRd<0b1001, 0b0101010, - (outs GPR8 - : $rd), - (ins GPR8 - : $src), - "dec\t$rd", [(set i8 - : $rd, (add i8 - : $src, -1)), - (implicit SREG)]>; + def DECRd : FRd<0b1001, 0b0101010, (outs GPR8:$rd), (ins GPR8:$src), + "dec\t$rd", + [(set i8:$rd, (add i8:$src, -1)), (implicit SREG)]>; } //===----------------------------------------------------------------------===// @@ -646,58 +499,35 @@ let isCommutable = 1, Defs = [R1, R0, SREG] in { // MUL Rd, Rr // Multiplies Rd by Rr and places the result into R1:R0. let usesCustomInserter = 1 in { - def MULRdRr : FRdRr<0b1001, 0b11, (outs), - (ins GPR8 - : $rd, GPR8 - : $rr), - "mul\t$rd, $rr", - [/*(set R1, R0, (smullohi i8:$rd, i8:$rr))*/]>, + def MULRdRr : FRdRr<0b1001, 0b11, (outs), (ins GPR8:$rd, GPR8:$rr), + "mul\t$rd, $rr", []>, Requires<[SupportsMultiplication]>; - def MULSRdRr : FMUL2RdRr<0, (outs), - (ins LD8 - : $rd, LD8 - : $rr), + def MULSRdRr : FMUL2RdRr<0, (outs), (ins LD8:$rd, LD8:$rr), "muls\t$rd, $rr", []>, Requires<[SupportsMultiplication]>; } - def MULSURdRr : FMUL2RdRr<1, (outs), - (ins LD8lo - : $rd, LD8lo - : $rr), + def MULSURdRr : FMUL2RdRr<1, (outs), (ins LD8lo:$rd, LD8lo:$rr), "mulsu\t$rd, $rr", []>, Requires<[SupportsMultiplication]>; - def FMUL : FFMULRdRr<0b01, (outs), - (ins LD8lo - : $rd, LD8lo - : $rr), + def FMUL : FFMULRdRr<0b01, (outs), (ins LD8lo:$rd, LD8lo:$rr), "fmul\t$rd, $rr", []>, Requires<[SupportsMultiplication]>; - def FMULS : FFMULRdRr<0b10, (outs), - (ins LD8lo - : $rd, LD8lo - : $rr), + def FMULS : FFMULRdRr<0b10, (outs), (ins LD8lo:$rd, LD8lo:$rr), "fmuls\t$rd, $rr", []>, Requires<[SupportsMultiplication]>; - def FMULSU : FFMULRdRr<0b11, (outs), - (ins LD8lo - : $rd, LD8lo - : $rr), + def FMULSU : FFMULRdRr<0b11, (outs), (ins LD8lo:$rd, LD8lo:$rr), "fmulsu\t$rd, $rr", []>, Requires<[SupportsMultiplication]>; } let Defs = - [R15, R14, R13, R12, R11, R10, R9, R8, R7, R6, R5, R4, R3, R2, R1, - R0] in def DESK : FDES<(outs), - (ins i8imm - : $k), - "des\t$k", []>, - Requires<[HasDES]>; + [R15, R14, R13, R12, R11, R10, R9, R8, R7, R6, R5, R4, R3, R2, R1, R0] in +def DESK : FDES<(outs), (ins i8imm:$k), "des\t$k", []>, Requires<[HasDES]>; //===----------------------------------------------------------------------===// // Logic -- GitLab From 372c275800140f35a697f12a2e83d94d5603eaf5 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Mon, 1 Apr 2024 17:28:44 -0700 Subject: [PATCH 0153/1690] [dfsan][test] Disable the test with internal_symbolizer After #87191 we had to add 8b135a7d1f59a5a7adccb162abf92d751209afe7, which makes symbolizer to calls a global constructor with `realloc`. --- compiler-rt/test/dfsan/mmap_at_init.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compiler-rt/test/dfsan/mmap_at_init.c b/compiler-rt/test/dfsan/mmap_at_init.c index a8d7535df4a6..9129dc7d3903 100644 --- a/compiler-rt/test/dfsan/mmap_at_init.c +++ b/compiler-rt/test/dfsan/mmap_at_init.c @@ -4,6 +4,9 @@ // // Tests that calling mmap() during during dfsan initialization works. +// `internal_symbolizer` can not use `realloc` on memory from the test `calloc`. +// UNSUPPORTED: internal_symbolizer + #include #include #include -- GitLab From f33a6dcf959238e82f6ad45333e3547d8cfcfe38 Mon Sep 17 00:00:00 2001 From: Chen Zheng Date: Tue, 2 Apr 2024 08:40:28 +0800 Subject: [PATCH 0154/1690] [PPC][NFC] add an option for GatherAllAliasesMaxDepth (#87071) GatherAllAliases is time consuming. Add an debug option on PPC to control the complexity of the function. This is useful when debuging compile time related issues. --- llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 7436b202fba0..43e4a34a9b34 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -137,6 +137,10 @@ static cl::opt PPCMinimumJumpTableEntries( "ppc-min-jump-table-entries", cl::init(64), cl::Hidden, cl::desc("Set minimum number of entries to use a jump table on PPC")); +static cl::opt PPCGatherAllAliasesMaxDepth( + "ppc-gather-alias-max-depth", cl::init(18), cl::Hidden, + cl::desc("max depth when checking alias info in GatherAllAliases()")); + STATISTIC(NumTailCalls, "Number of tail calls"); STATISTIC(NumSiblingCalls, "Number of sibling calls"); STATISTIC(ShufflesHandledWithVPERM, @@ -1512,6 +1516,8 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, // than the corresponding branch. This information is used in CGP to decide // when to convert selects into branches. PredictableSelectIsExpensive = Subtarget.isPredictableSelectIsExpensive(); + + GatherAllAliasesMaxDepth = PPCGatherAllAliasesMaxDepth; } // *********************************** NOTE ************************************ -- GitLab From 84f24c2daffc40fc10b4ea2ae69016ebdabfc0ed Mon Sep 17 00:00:00 2001 From: Shih-Po Hung Date: Tue, 2 Apr 2024 09:26:27 +0800 Subject: [PATCH 0155/1690] [RISCV][TTI] Scale the cost of intrinsic umin/umax/smin/smax with LMUL (#87245) Use the return type to measure the LMUL size for throughput/latency cost --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 22 +++- .../Analysis/CostModel/RISCV/int-min-max.ll | 120 +++++++++--------- 2 files changed, 80 insertions(+), 62 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index efcaa65605e0..ed4b0ca8c941 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -810,9 +810,27 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, case Intrinsic::smin: case Intrinsic::smax: { auto LT = getTypeLegalizationCost(RetTy); - if ((ST->hasVInstructions() && LT.second.isVector()) || - (LT.second.isScalarInteger() && ST->hasStdExtZbb())) + if (LT.second.isScalarInteger() && ST->hasStdExtZbb()) return LT.first; + + if (ST->hasVInstructions() && LT.second.isVector()) { + unsigned Op; + switch (ICA.getID()) { + case Intrinsic::umin: + Op = RISCV::VMINU_VV; + break; + case Intrinsic::umax: + Op = RISCV::VMAXU_VV; + break; + case Intrinsic::smin: + Op = RISCV::VMIN_VV; + break; + case Intrinsic::smax: + Op = RISCV::VMAX_VV; + break; + } + return LT.first * getRISCVInstructionCost(Op, LT.second, CostKind); + } break; } case Intrinsic::sadd_sat: diff --git a/llvm/test/Analysis/CostModel/RISCV/int-min-max.ll b/llvm/test/Analysis/CostModel/RISCV/int-min-max.ll index ec669c986c15..79cf1c84ed49 100644 --- a/llvm/test/Analysis/CostModel/RISCV/int-min-max.ll +++ b/llvm/test/Analysis/CostModel/RISCV/int-min-max.ll @@ -12,36 +12,36 @@ define void @smax() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = call @llvm.smax.nxv2i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = call @llvm.smax.nxv4i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = call @llvm.smax.nxv8i8( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = call @llvm.smax.nxv16i8( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = call @llvm.smax.nxv16i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = call i16 @llvm.smax.i16(i16 undef, i16 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %15 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = call @llvm.smax.nxv1i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %17 = call @llvm.smax.nxv2i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %18 = call @llvm.smax.nxv4i16( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = call @llvm.smax.nxv8i16( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = call @llvm.smax.nxv16i16( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %19 = call @llvm.smax.nxv8i16( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %20 = call @llvm.smax.nxv16i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %21 = call i32 @llvm.smax.i32(i32 undef, i32 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %22 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %23 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %24 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %25 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %24 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %25 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %26 = call @llvm.smax.nxv1i32( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %27 = call @llvm.smax.nxv2i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %28 = call @llvm.smax.nxv4i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %29 = call @llvm.smax.nxv8i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %30 = call @llvm.smax.nxv16i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %28 = call @llvm.smax.nxv4i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %29 = call @llvm.smax.nxv8i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %30 = call @llvm.smax.nxv16i32( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %31 = call i64 @llvm.smax.i64(i64 undef, i64 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %32 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %33 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %34 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %35 = call <16 x i64> @llvm.smax.v16i64(<16 x i64> undef, <16 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %33 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %34 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %35 = call <16 x i64> @llvm.smax.v16i64(<16 x i64> undef, <16 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %36 = call @llvm.smax.nxv1i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %37 = call @llvm.smax.nxv2i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %38 = call @llvm.smax.nxv4i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %39 = call @llvm.smax.nxv8i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %37 = call @llvm.smax.nxv2i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %38 = call @llvm.smax.nxv4i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %39 = call @llvm.smax.nxv8i64( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; call i8 @llvm.smax.i8(i8 undef, i8 undef) @@ -97,36 +97,36 @@ define void @smin() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = call @llvm.smin.nxv2i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = call @llvm.smin.nxv4i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = call @llvm.smin.nxv8i8( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = call @llvm.smin.nxv16i8( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = call @llvm.smin.nxv16i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = call i16 @llvm.smin.i16(i16 undef, i16 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %15 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = call @llvm.smin.nxv1i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %17 = call @llvm.smin.nxv2i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %18 = call @llvm.smin.nxv4i16( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = call @llvm.smin.nxv8i16( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = call @llvm.smin.nxv16i16( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %19 = call @llvm.smin.nxv8i16( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %20 = call @llvm.smin.nxv16i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %21 = call i32 @llvm.smin.i32(i32 undef, i32 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %22 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %23 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %24 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %25 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %24 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %25 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %26 = call @llvm.smin.nxv1i32( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %27 = call @llvm.smin.nxv2i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %28 = call @llvm.smin.nxv4i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %29 = call @llvm.smin.nxv8i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %30 = call @llvm.smin.nxv16i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %28 = call @llvm.smin.nxv4i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %29 = call @llvm.smin.nxv8i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %30 = call @llvm.smin.nxv16i32( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %31 = call i64 @llvm.smin.i64(i64 undef, i64 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %32 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %33 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %34 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %35 = call <16 x i64> @llvm.smin.v16i64(<16 x i64> undef, <16 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %33 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %34 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %35 = call <16 x i64> @llvm.smin.v16i64(<16 x i64> undef, <16 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %36 = call @llvm.smin.nxv1i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %37 = call @llvm.smin.nxv2i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %38 = call @llvm.smin.nxv4i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %39 = call @llvm.smin.nxv8i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %37 = call @llvm.smin.nxv2i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %38 = call @llvm.smin.nxv4i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %39 = call @llvm.smin.nxv8i64( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; call i8 @llvm.smin.i8(i8 undef, i8 undef) @@ -182,36 +182,36 @@ define void @umax() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = call @llvm.umax.nxv2i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = call @llvm.umax.nxv4i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = call @llvm.umax.nxv8i8( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = call @llvm.umax.nxv16i8( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = call @llvm.umax.nxv16i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = call i16 @llvm.umax.i16(i16 undef, i16 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %15 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = call @llvm.umax.nxv1i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %17 = call @llvm.umax.nxv2i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %18 = call @llvm.umax.nxv4i16( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = call @llvm.umax.nxv8i16( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = call @llvm.umax.nxv16i16( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %19 = call @llvm.umax.nxv8i16( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %20 = call @llvm.umax.nxv16i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %21 = call i32 @llvm.umax.i32(i32 undef, i32 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %22 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %23 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %24 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %25 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %24 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %25 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %26 = call @llvm.umax.nxv1i32( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %27 = call @llvm.umax.nxv2i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %28 = call @llvm.umax.nxv4i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %29 = call @llvm.umax.nxv8i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %30 = call @llvm.umax.nxv16i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %28 = call @llvm.umax.nxv4i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %29 = call @llvm.umax.nxv8i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %30 = call @llvm.umax.nxv16i32( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %31 = call i64 @llvm.umax.i64(i64 undef, i64 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %32 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %33 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %34 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %35 = call <16 x i64> @llvm.umax.v16i64(<16 x i64> undef, <16 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %33 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %34 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %35 = call <16 x i64> @llvm.umax.v16i64(<16 x i64> undef, <16 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %36 = call @llvm.umax.nxv1i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %37 = call @llvm.umax.nxv2i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %38 = call @llvm.umax.nxv4i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %39 = call @llvm.umax.nxv8i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %37 = call @llvm.umax.nxv2i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %38 = call @llvm.umax.nxv4i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %39 = call @llvm.umax.nxv8i64( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; call i8 @llvm.umax.i8(i8 undef, i8 undef) @@ -267,36 +267,36 @@ define void @umin() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = call @llvm.umin.nxv2i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = call @llvm.umin.nxv4i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = call @llvm.umin.nxv8i8( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = call @llvm.umin.nxv16i8( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = call @llvm.umin.nxv16i8( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = call i16 @llvm.umin.i16(i16 undef, i16 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %15 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = call @llvm.umin.nxv1i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %17 = call @llvm.umin.nxv2i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %18 = call @llvm.umin.nxv4i16( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = call @llvm.umin.nxv8i16( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = call @llvm.umin.nxv16i16( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %19 = call @llvm.umin.nxv8i16( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %20 = call @llvm.umin.nxv16i16( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %21 = call i32 @llvm.umin.i32(i32 undef, i32 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %22 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %23 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %24 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %25 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %24 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %25 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %26 = call @llvm.umin.nxv1i32( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %27 = call @llvm.umin.nxv2i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %28 = call @llvm.umin.nxv4i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %29 = call @llvm.umin.nxv8i32( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %30 = call @llvm.umin.nxv16i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %28 = call @llvm.umin.nxv4i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %29 = call @llvm.umin.nxv8i32( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %30 = call @llvm.umin.nxv16i32( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %31 = call i64 @llvm.umin.i64(i64 undef, i64 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %32 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %33 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %34 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %35 = call <16 x i64> @llvm.umin.v16i64(<16 x i64> undef, <16 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %33 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %34 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %35 = call <16 x i64> @llvm.umin.v16i64(<16 x i64> undef, <16 x i64> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %36 = call @llvm.umin.nxv1i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %37 = call @llvm.umin.nxv2i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %38 = call @llvm.umin.nxv4i64( undef, undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %39 = call @llvm.umin.nxv8i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %37 = call @llvm.umin.nxv2i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %38 = call @llvm.umin.nxv4i64( undef, undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %39 = call @llvm.umin.nxv8i64( undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; call i8 @llvm.umin.i8(i8 undef, i8 undef) -- GitLab From 30fd099d5062638b5fe6b89135ad6433a888023a Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Mon, 1 Apr 2024 18:30:23 -0700 Subject: [PATCH 0156/1690] [InstallAPI] Fixup dsym test (#87299) Update the test to run when the compiler is built to support arm64-darwin targets. --- clang/test/InstallAPI/diagnostics-dsym.test | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/clang/test/InstallAPI/diagnostics-dsym.test b/clang/test/InstallAPI/diagnostics-dsym.test index 8a1b394f2f86..c9cbeffef7ba 100644 --- a/clang/test/InstallAPI/diagnostics-dsym.test +++ b/clang/test/InstallAPI/diagnostics-dsym.test @@ -1,23 +1,24 @@ -; REQUIRES: 86_64-darwin +; REQUIRES: system-darwin +; REQUIRES: target-aarch64 ; RUN: rm -rf %t ; RUN: split-file %s %t // Build a simple dylib with debug info. -; RUN: %clang --target=x86_64-apple-macos10.15 -g -dynamiclib %t/foo.c \ +; RUN: %clang --target=arm64-apple-macos11 -g -dynamiclib %t/foo.c \ ; RUN: -current_version 1 -compatibility_version 1 -L%t/usr/lib \ ; RUN: -save-temps \ ; RUN: -o %t/foo.dylib -install_name %t/foo.dylib ; RUN: dsymutil %t/foo.dylib -o %t/foo.dSYM -; RUN: not clang-installapi -x c++ --target=x86_64-apple-macos10.15 \ +; RUN: not clang-installapi -x c++ --target=arm64-apple-macos11 \ ; RUN: -install_name %t/foo.dylib \ ; RUN: -current_version 1 -compatibility_version 1 \ ; RUN: -o %t/output.tbd \ ; RUN: --verify-against=%t/foo.dylib --dsym=%t/foo.dSYM \ ; RUN: --verify-mode=Pedantic 2>&1 | FileCheck %s -; CHECK: violations found for x86_64 +; CHECK: violations found for arm64 ; CHECK: foo.c:5:0: error: no declaration found for exported symbol 'bar' in dynamic library ; CHECK: foo.c:1:0: error: no declaration found for exported symbol 'foo' in dynamic library @@ -31,9 +32,9 @@ char bar = 'a'; ;--- usr/lib/libSystem.tbd --- !tapi-tbd tbd-version: 4 -targets: [ x86_64-macos ] +targets: [ arm64-macos ] install-name: '/usr/lib/libSystem.B.dylib' exports: - - targets: [ x86_64-macos ] + - targets: [ arm64-macos ] symbols: [ dyld_stub_binder ] ... -- GitLab From d7a43a00fe80007de5d7614576b180d3d21d541b Mon Sep 17 00:00:00 2001 From: Shih-Po Hung Date: Tue, 2 Apr 2024 09:30:51 +0800 Subject: [PATCH 0157/1690] [RISCV][TTI] Scale the cost of trunc/fptrunc/fpext with LMUL (#87101) Use the destination data type to measure the LMUL size for latency/throughput cost --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 25 +- llvm/test/Analysis/CostModel/RISCV/cast.ll | 454 +++++++++--------- .../CostModel/RISCV/reduce-scalable-fp.ll | 12 +- .../CostModel/RISCV/rvv-insertelement.ll | 84 ++-- .../CostModel/RISCV/shuffle-broadcast.ll | 2 +- 5 files changed, 298 insertions(+), 279 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index ed4b0ca8c941..38304ff90252 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -927,6 +927,7 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, if (!IsTypeLegal) return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); + std::pair SrcLT = getTypeLegalizationCost(Src); std::pair DstLT = getTypeLegalizationCost(Dst); int ISD = TLI->InstructionOpcodeToISD(Opcode); @@ -961,13 +962,31 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, // Instead we use the following instructions to truncate to mask vector: // vand.vi v8, v8, 1 // vmsne.vi v0, v8, 0 - return 2; + return getRISCVInstructionCost({RISCV::VAND_VI, RISCV::VMSNE_VI}, + SrcLT.second, CostKind); } [[fallthrough]]; case ISD::FP_EXTEND: - case ISD::FP_ROUND: + case ISD::FP_ROUND: { // Counts of narrow/widen instructions. - return std::abs(PowDiff); + unsigned SrcEltSize = Src->getScalarSizeInBits(); + unsigned DstEltSize = Dst->getScalarSizeInBits(); + + unsigned Op = (ISD == ISD::TRUNCATE) ? RISCV::VNSRL_WI + : (ISD == ISD::FP_EXTEND) ? RISCV::VFWCVT_F_F_V + : RISCV::VFNCVT_F_F_W; + InstructionCost Cost = 0; + for (; SrcEltSize != DstEltSize;) { + MVT ElementMVT = (ISD == ISD::TRUNCATE) + ? MVT::getIntegerVT(DstEltSize) + : MVT::getFloatingPointVT(DstEltSize); + MVT DstMVT = DstLT.second.changeVectorElementType(ElementMVT); + DstEltSize = + (DstEltSize > SrcEltSize) ? DstEltSize >> 1 : DstEltSize << 1; + Cost += getRISCVInstructionCost(Op, DstMVT, CostKind); + } + return Cost; + } case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: case ISD::SINT_TO_FP: diff --git a/llvm/test/Analysis/CostModel/RISCV/cast.ll b/llvm/test/Analysis/CostModel/RISCV/cast.ll index 14da9a3f79d7..6ddd57a24c51 100644 --- a/llvm/test/Analysis/CostModel/RISCV/cast.ll +++ b/llvm/test/Analysis/CostModel/RISCV/cast.ll @@ -1035,17 +1035,17 @@ define void @trunc() { ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i1 = trunc <4 x i8> undef to <4 x i1> ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i1 = trunc <4 x i16> undef to <4 x i1> ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i1 = trunc <4 x i32> undef to <4 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_v4i1 = trunc <4 x i64> undef to <4 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64_v4i1 = trunc <4 x i64> undef to <4 x i1> ; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i8 = trunc <8 x i16> undef to <8 x i8> ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i32_v8i8 = trunc <8 x i32> undef to <8 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8i64_v8i8 = trunc <8 x i64> undef to <8 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_v8i8 = trunc <8 x i64> undef to <8 x i8> ; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i16 = trunc <8 x i32> undef to <8 x i16> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i64_v8i16 = trunc <8 x i64> undef to <8 x i16> -; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i64_v8i32 = trunc <8 x i64> undef to <8 x i32> +; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8i64_v8i16 = trunc <8 x i64> undef to <8 x i16> +; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i64_v8i32 = trunc <8 x i64> undef to <8 x i32> ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i1 = trunc <8 x i8> undef to <8 x i1> ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i1 = trunc <8 x i16> undef to <8 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i32_v8i1 = trunc <8 x i32> undef to <8 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i64_v8i1 = trunc <8 x i64> undef to <8 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i1 = trunc <8 x i32> undef to <8 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64_v8i1 = trunc <8 x i64> undef to <8 x i1> ; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i8 = trunc <2 x i16> undef to <2 x i8> ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_v16i8 = trunc <2 x i32> undef to <2 x i8> ; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v16i64_v16i8 = trunc <2 x i64> undef to <2 x i8> @@ -1057,44 +1057,44 @@ define void @trunc() { ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_v16i1 = trunc <2 x i32> undef to <2 x i1> ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i64_v16i1 = trunc <2 x i64> undef to <2 x i1> ; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i8 = trunc <16 x i16> undef to <16 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i32_v32i8 = trunc <16 x i32> undef to <16 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v32i64_v32i8 = trunc <16 x i64> undef to <16 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i32_v32i16 = trunc <16 x i32> undef to <16 x i16> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i64_v32i16 = trunc <16 x i64> undef to <16 x i16> -; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i64_v32i32 = trunc <16 x i64> undef to <16 x i32> +; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i8 = trunc <16 x i32> undef to <16 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v32i64_v32i8 = trunc <16 x i64> undef to <16 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i32_v32i16 = trunc <16 x i32> undef to <16 x i16> +; RV32-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v32i64_v32i16 = trunc <16 x i64> undef to <16 x i16> +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i64_v32i32 = trunc <16 x i64> undef to <16 x i32> ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i8_v32i1 = trunc <16 x i8> undef to <16 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i16_v32i1 = trunc <16 x i16> undef to <16 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i32_v32i1 = trunc <16 x i32> undef to <16 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i64_v32i1 = trunc <16 x i64> undef to <16 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v64i16_v64i8 = trunc <64 x i16> undef to <64 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v64i32_v64i8 = trunc <64 x i32> undef to <64 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v64i64_v64i8 = trunc <64 x i64> undef to <64 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v64i32_v64i16 = trunc <64 x i32> undef to <64 x i16> -; RV32-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v64i64_v64i16 = trunc <64 x i64> undef to <64 x i16> -; RV32-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v64i64_v64i32 = trunc <64 x i64> undef to <64 x i32> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v64i8_v64i1 = trunc <64 x i8> undef to <64 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v64i16_v64i1 = trunc <64 x i16> undef to <64 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v64i32_v64i1 = trunc <64 x i32> undef to <64 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i16_v32i1 = trunc <16 x i16> undef to <16 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v32i32_v32i1 = trunc <16 x i32> undef to <16 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v32i64_v32i1 = trunc <16 x i64> undef to <16 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v64i16_v64i8 = trunc <64 x i16> undef to <64 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %v64i32_v64i8 = trunc <64 x i32> undef to <64 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %v64i64_v64i8 = trunc <64 x i64> undef to <64 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v64i32_v64i16 = trunc <64 x i32> undef to <64 x i16> +; RV32-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %v64i64_v64i16 = trunc <64 x i64> undef to <64 x i16> +; RV32-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v64i64_v64i32 = trunc <64 x i64> undef to <64 x i32> +; RV32-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i1 = trunc <64 x i8> undef to <64 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v64i16_v64i1 = trunc <64 x i16> undef to <64 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v64i32_v64i1 = trunc <64 x i32> undef to <64 x i1> ; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v64i64_v64i1 = trunc <64 x i64> undef to <64 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v128i16_v128i8 = trunc <128 x i16> undef to <128 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v128i32_v128i8 = trunc <128 x i32> undef to <128 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %v128i64_v128i8 = trunc <128 x i64> undef to <128 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v128i32_v128i16 = trunc <128 x i32> undef to <128 x i16> -; RV32-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v128i64_v128i16 = trunc <128 x i64> undef to <128 x i16> -; RV32-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v128i64_v128i32 = trunc <128 x i64> undef to <128 x i32> -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v128i8_v128i1 = trunc <128 x i8> undef to <128 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v128i16_v128i1 = trunc <128 x i16> undef to <128 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v128i32_v128i1 = trunc <128 x i32> undef to <128 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v128i16_v128i8 = trunc <128 x i16> undef to <128 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %v128i32_v128i8 = trunc <128 x i32> undef to <128 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 63 for instruction: %v128i64_v128i8 = trunc <128 x i64> undef to <128 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v128i32_v128i16 = trunc <128 x i32> undef to <128 x i16> +; RV32-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %v128i64_v128i16 = trunc <128 x i64> undef to <128 x i16> +; RV32-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v128i64_v128i32 = trunc <128 x i64> undef to <128 x i32> +; RV32-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v128i8_v128i1 = trunc <128 x i8> undef to <128 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v128i16_v128i1 = trunc <128 x i16> undef to <128 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 67 for instruction: %v128i32_v128i1 = trunc <128 x i32> undef to <128 x i1> ; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v128i64_v128i1 = trunc <128 x i64> undef to <128 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v256i16_v256i8 = trunc <256 x i16> undef to <256 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v256i32_v256i8 = trunc <256 x i32> undef to <256 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %v256i64_v256i8 = trunc <256 x i64> undef to <256 x i8> -; RV32-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v256i32_v256i16 = trunc <256 x i32> undef to <256 x i16> -; RV32-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %v256i64_v256i16 = trunc <256 x i64> undef to <256 x i16> -; RV32-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v256i64_v256i32 = trunc <256 x i64> undef to <256 x i32> -; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v256i8_v256i1 = trunc <256 x i8> undef to <256 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v256i16_v256i1 = trunc <256 x i16> undef to <256 x i1> -; RV32-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v256i32_v256i1 = trunc <256 x i32> undef to <256 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v256i16_v256i8 = trunc <256 x i16> undef to <256 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %v256i32_v256i8 = trunc <256 x i32> undef to <256 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 126 for instruction: %v256i64_v256i8 = trunc <256 x i64> undef to <256 x i8> +; RV32-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v256i32_v256i16 = trunc <256 x i32> undef to <256 x i16> +; RV32-NEXT: Cost Model: Found an estimated cost of 108 for instruction: %v256i64_v256i16 = trunc <256 x i64> undef to <256 x i16> +; RV32-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v256i64_v256i32 = trunc <256 x i64> undef to <256 x i32> +; RV32-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v256i8_v256i1 = trunc <256 x i8> undef to <256 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 66 for instruction: %v256i16_v256i1 = trunc <256 x i16> undef to <256 x i1> +; RV32-NEXT: Cost Model: Found an estimated cost of 134 for instruction: %v256i32_v256i1 = trunc <256 x i32> undef to <256 x i1> ; RV32-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v256i64_v256i1 = trunc <256 x i64> undef to <256 x i1> ; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16_nxv1i8 = trunc undef to ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1i32_nxv1i8 = trunc undef to @@ -1115,56 +1115,56 @@ define void @trunc() { ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i1 = trunc undef to ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i1 = trunc undef to ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64_nxv2i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv2i64_nxv2i1 = trunc undef to ; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i8 = trunc undef to ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32_nxv4i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4i64_nxv4i8 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64_nxv4i8 = trunc undef to ; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i16 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i64_nxv4i16 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64_nxv4i32 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4i64_nxv4i16 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i64_nxv4i32 = trunc undef to ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i1 = trunc undef to ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32_nxv4i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i64_nxv4i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv4i64_nxv4i1 = trunc undef to ; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i32_nxv8i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8i64_nxv8i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i16 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i64_nxv8i16 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64_nxv8i32 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8i32_nxv8i8 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv8i64_nxv8i8 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i32_nxv8i16 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv8i64_nxv8i16 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8i64_nxv8i32 = trunc undef to ; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16_nxv8i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i32_nxv8i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i64_nxv8i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv16i32_nxv16i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv16i64_nxv16i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32_nxv16i16 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16i64_nxv16i16 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv16i64_nxv16i32 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8_nxv16i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv16i16_nxv16i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv16i32_nxv16i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16i64_nxv16i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv32i16_nxv32i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv32i32_nxv32i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %nxv32i64_nxv32i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv32i32_nxv32i16 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv32i64_nxv32i16 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv32i64_nxv32i32 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv32i8_nxv32i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv32i16_nxv32i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv32i32_nxv32i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv32i64_nxv32i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv64i16_nxv64i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv64i32_nxv64i8 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %nxv8i64_nxv8i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv16i16_nxv16i8 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv16i32_nxv16i8 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %nxv16i64_nxv16i8 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16i32_nxv16i16 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %nxv16i64_nxv16i16 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16i64_nxv16i32 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %nxv16i32_nxv16i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %nxv16i64_nxv16i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv32i16_nxv32i8 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %nxv32i32_nxv32i8 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %nxv32i64_nxv32i8 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv32i32_nxv32i16 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %nxv32i64_nxv32i16 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %nxv32i64_nxv32i32 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %nxv32i16_nxv32i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %nxv32i32_nxv32i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 67 for instruction: %nxv32i64_nxv32i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv64i16_nxv64i8 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %nxv64i32_nxv64i8 = trunc undef to ; RV32-NEXT: Cost Model: Invalid cost for instruction: %nxv64i64_nxv64i8 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv64i32_nxv64i16 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %nxv64i64_nxv64i16 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %nxv64i64_nxv64i32 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv64i8_nxv64i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv64i16_nxv64i1 = trunc undef to -; RV32-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv64i32_nxv64i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %nxv64i32_nxv64i16 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 55 for instruction: %nxv64i64_nxv64i16 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 37 for instruction: %nxv64i64_nxv64i32 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %nxv64i8_nxv64i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %nxv64i16_nxv64i1 = trunc undef to +; RV32-NEXT: Cost Model: Found an estimated cost of 67 for instruction: %nxv64i32_nxv64i1 = trunc undef to ; RV32-NEXT: Cost Model: Invalid cost for instruction: %nxv64i64_nxv64i1 = trunc undef to ; RV32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; @@ -1188,17 +1188,17 @@ define void @trunc() { ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i1 = trunc <4 x i8> undef to <4 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i1 = trunc <4 x i16> undef to <4 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i1 = trunc <4 x i32> undef to <4 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_v4i1 = trunc <4 x i64> undef to <4 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64_v4i1 = trunc <4 x i64> undef to <4 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i8 = trunc <8 x i16> undef to <8 x i8> ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i32_v8i8 = trunc <8 x i32> undef to <8 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8i64_v8i8 = trunc <8 x i64> undef to <8 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_v8i8 = trunc <8 x i64> undef to <8 x i8> ; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i16 = trunc <8 x i32> undef to <8 x i16> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i64_v8i16 = trunc <8 x i64> undef to <8 x i16> -; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i64_v8i32 = trunc <8 x i64> undef to <8 x i32> +; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8i64_v8i16 = trunc <8 x i64> undef to <8 x i16> +; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i64_v8i32 = trunc <8 x i64> undef to <8 x i32> ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i1 = trunc <8 x i8> undef to <8 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i1 = trunc <8 x i16> undef to <8 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i32_v8i1 = trunc <8 x i32> undef to <8 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i64_v8i1 = trunc <8 x i64> undef to <8 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i1 = trunc <8 x i32> undef to <8 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64_v8i1 = trunc <8 x i64> undef to <8 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i8 = trunc <2 x i16> undef to <2 x i8> ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_v16i8 = trunc <2 x i32> undef to <2 x i8> ; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v16i64_v16i8 = trunc <2 x i64> undef to <2 x i8> @@ -1210,43 +1210,43 @@ define void @trunc() { ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_v16i1 = trunc <2 x i32> undef to <2 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i64_v16i1 = trunc <2 x i64> undef to <2 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i8 = trunc <16 x i16> undef to <16 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i32_v32i8 = trunc <16 x i32> undef to <16 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v32i64_v32i8 = trunc <16 x i64> undef to <16 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i32_v32i16 = trunc <16 x i32> undef to <16 x i16> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i64_v32i16 = trunc <16 x i64> undef to <16 x i16> -; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i64_v32i32 = trunc <16 x i64> undef to <16 x i32> +; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i8 = trunc <16 x i32> undef to <16 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v32i64_v32i8 = trunc <16 x i64> undef to <16 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i32_v32i16 = trunc <16 x i32> undef to <16 x i16> +; RV64-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v32i64_v32i16 = trunc <16 x i64> undef to <16 x i16> +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i64_v32i32 = trunc <16 x i64> undef to <16 x i32> ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i8_v32i1 = trunc <16 x i8> undef to <16 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i16_v32i1 = trunc <16 x i16> undef to <16 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i32_v32i1 = trunc <16 x i32> undef to <16 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i64_v32i1 = trunc <16 x i64> undef to <16 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v64i16_v64i8 = trunc <64 x i16> undef to <64 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v64i32_v64i8 = trunc <64 x i32> undef to <64 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v64i64_v64i8 = trunc <64 x i64> undef to <64 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v64i32_v64i16 = trunc <64 x i32> undef to <64 x i16> -; RV64-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v64i64_v64i16 = trunc <64 x i64> undef to <64 x i16> -; RV64-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v64i64_v64i32 = trunc <64 x i64> undef to <64 x i32> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v64i8_v64i1 = trunc <64 x i8> undef to <64 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v64i16_v64i1 = trunc <64 x i16> undef to <64 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i16_v32i1 = trunc <16 x i16> undef to <16 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v32i32_v32i1 = trunc <16 x i32> undef to <16 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v32i64_v32i1 = trunc <16 x i64> undef to <16 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v64i16_v64i8 = trunc <64 x i16> undef to <64 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %v64i32_v64i8 = trunc <64 x i32> undef to <64 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %v64i64_v64i8 = trunc <64 x i64> undef to <64 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v64i32_v64i16 = trunc <64 x i32> undef to <64 x i16> +; RV64-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %v64i64_v64i16 = trunc <64 x i64> undef to <64 x i16> +; RV64-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v64i64_v64i32 = trunc <64 x i64> undef to <64 x i32> +; RV64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i1 = trunc <64 x i8> undef to <64 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v64i16_v64i1 = trunc <64 x i16> undef to <64 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v64i32_v64i1 = trunc <64 x i32> undef to <64 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v64i64_v64i1 = trunc <64 x i64> undef to <64 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v128i16_v128i8 = trunc <128 x i16> undef to <128 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v128i32_v128i8 = trunc <128 x i32> undef to <128 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %v128i64_v128i8 = trunc <128 x i64> undef to <128 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v128i32_v128i16 = trunc <128 x i32> undef to <128 x i16> -; RV64-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v128i64_v128i16 = trunc <128 x i64> undef to <128 x i16> -; RV64-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v128i64_v128i32 = trunc <128 x i64> undef to <128 x i32> -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v128i8_v128i1 = trunc <128 x i8> undef to <128 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v128i16_v128i1 = trunc <128 x i16> undef to <128 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v128i16_v128i8 = trunc <128 x i16> undef to <128 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %v128i32_v128i8 = trunc <128 x i32> undef to <128 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 63 for instruction: %v128i64_v128i8 = trunc <128 x i64> undef to <128 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v128i32_v128i16 = trunc <128 x i32> undef to <128 x i16> +; RV64-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %v128i64_v128i16 = trunc <128 x i64> undef to <128 x i16> +; RV64-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v128i64_v128i32 = trunc <128 x i64> undef to <128 x i32> +; RV64-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v128i8_v128i1 = trunc <128 x i8> undef to <128 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v128i16_v128i1 = trunc <128 x i16> undef to <128 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v128i32_v128i1 = trunc <128 x i32> undef to <128 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v128i64_v128i1 = trunc <128 x i64> undef to <128 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v256i16_v256i8 = trunc <256 x i16> undef to <256 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v256i32_v256i8 = trunc <256 x i32> undef to <256 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %v256i64_v256i8 = trunc <256 x i64> undef to <256 x i8> -; RV64-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v256i32_v256i16 = trunc <256 x i32> undef to <256 x i16> -; RV64-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %v256i64_v256i16 = trunc <256 x i64> undef to <256 x i16> -; RV64-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v256i64_v256i32 = trunc <256 x i64> undef to <256 x i32> -; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v256i8_v256i1 = trunc <256 x i8> undef to <256 x i1> -; RV64-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v256i16_v256i1 = trunc <256 x i16> undef to <256 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v256i16_v256i8 = trunc <256 x i16> undef to <256 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %v256i32_v256i8 = trunc <256 x i32> undef to <256 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 126 for instruction: %v256i64_v256i8 = trunc <256 x i64> undef to <256 x i8> +; RV64-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v256i32_v256i16 = trunc <256 x i32> undef to <256 x i16> +; RV64-NEXT: Cost Model: Found an estimated cost of 108 for instruction: %v256i64_v256i16 = trunc <256 x i64> undef to <256 x i16> +; RV64-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v256i64_v256i32 = trunc <256 x i64> undef to <256 x i32> +; RV64-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v256i8_v256i1 = trunc <256 x i8> undef to <256 x i1> +; RV64-NEXT: Cost Model: Found an estimated cost of 66 for instruction: %v256i16_v256i1 = trunc <256 x i16> undef to <256 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v256i32_v256i1 = trunc <256 x i32> undef to <256 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v256i64_v256i1 = trunc <256 x i64> undef to <256 x i1> ; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16_nxv1i8 = trunc undef to @@ -1268,57 +1268,57 @@ define void @trunc() { ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i1 = trunc undef to ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i1 = trunc undef to ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64_nxv2i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv2i64_nxv2i1 = trunc undef to ; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i8 = trunc undef to ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32_nxv4i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4i64_nxv4i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64_nxv4i8 = trunc undef to ; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i16 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i64_nxv4i16 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64_nxv4i32 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4i64_nxv4i16 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i64_nxv4i32 = trunc undef to ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i1 = trunc undef to ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32_nxv4i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i64_nxv4i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv4i64_nxv4i1 = trunc undef to ; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i32_nxv8i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8i64_nxv8i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i16 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i64_nxv8i16 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64_nxv8i32 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8i32_nxv8i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv8i64_nxv8i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i32_nxv8i16 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv8i64_nxv8i16 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8i64_nxv8i32 = trunc undef to ; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16_nxv8i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i32_nxv8i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i64_nxv8i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv16i32_nxv16i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv16i64_nxv16i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32_nxv16i16 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16i64_nxv16i16 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv16i64_nxv16i32 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8_nxv16i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv16i16_nxv16i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv16i32_nxv16i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16i64_nxv16i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv32i16_nxv32i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv32i32_nxv32i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %nxv32i64_nxv32i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv32i32_nxv32i16 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv32i64_nxv32i16 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv32i64_nxv32i32 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv32i8_nxv32i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv32i16_nxv32i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv32i32_nxv32i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv32i64_nxv32i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv64i16_nxv64i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv64i32_nxv64i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %nxv64i64_nxv64i8 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv64i32_nxv64i16 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %nxv64i64_nxv64i16 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %nxv64i64_nxv64i32 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv64i8_nxv64i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv64i16_nxv64i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv64i32_nxv64i1 = trunc undef to -; RV64-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %nxv64i64_nxv64i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %nxv8i64_nxv8i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv16i16_nxv16i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv16i32_nxv16i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %nxv16i64_nxv16i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16i32_nxv16i16 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %nxv16i64_nxv16i16 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16i64_nxv16i32 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %nxv16i32_nxv16i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %nxv16i64_nxv16i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv32i16_nxv32i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %nxv32i32_nxv32i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %nxv32i64_nxv32i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv32i32_nxv32i16 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %nxv32i64_nxv32i16 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %nxv32i64_nxv32i32 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %nxv32i16_nxv32i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %nxv32i32_nxv32i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 67 for instruction: %nxv32i64_nxv32i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv64i16_nxv64i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %nxv64i32_nxv64i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 63 for instruction: %nxv64i64_nxv64i8 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %nxv64i32_nxv64i16 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %nxv64i64_nxv64i16 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %nxv64i64_nxv64i32 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %nxv64i8_nxv64i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %nxv64i16_nxv64i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 67 for instruction: %nxv64i32_nxv64i1 = trunc undef to +; RV64-NEXT: Cost Model: Found an estimated cost of 135 for instruction: %nxv64i64_nxv64i1 = trunc undef to ; RV64-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2i16_v2i8 = trunc <2 x i16> undef to <2 x i8> @@ -1495,44 +1495,44 @@ define void @fpext() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16_v2f64 = fpext <2 x half> undef to <2 x double> ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2f64 = fpext <2 x float> undef to <2 x double> ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16_v4f32 = fpext <4 x half> undef to <4 x float> -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f16_v4f64 = fpext <4 x half> undef to <4 x double> -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32_v4f64 = fpext <4 x float> undef to <4 x double> -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16_v8f32 = fpext <8 x half> undef to <8 x float> -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f16_v8f64 = fpext <8 x half> undef to <8 x double> -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32_v8f64 = fpext <8 x float> undef to <8 x double> -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16_v16f32 = fpext <16 x half> undef to <16 x float> -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16_v16f64 = fpext <16 x half> undef to <16 x double> -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32_v16f64 = fpext <16 x float> undef to <16 x double> -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32f16_v32f32 = fpext <32 x half> undef to <32 x float> -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v32f16_v32f64 = fpext <32 x half> undef to <32 x double> -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v32f32_v32f64 = fpext <32 x float> undef to <32 x double> -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v64f16_v64f32 = fpext <64 x half> undef to <64 x float> -; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v64f16_v64f64 = fpext <64 x half> undef to <64 x double> -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v64f32_v64f64 = fpext <64 x float> undef to <64 x double> -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v128f16_v128f32 = fpext <128 x half> undef to <128 x float> -; CHECK-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v128f16_v128f64 = fpext <128 x half> undef to <128 x double> -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v128f32_v128f64 = fpext <128 x float> undef to <128 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f16_v4f64 = fpext <4 x half> undef to <4 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f32_v4f64 = fpext <4 x float> undef to <4 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f16_v8f32 = fpext <8 x half> undef to <8 x float> +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8f16_v8f64 = fpext <8 x half> undef to <8 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f32_v8f64 = fpext <8 x float> undef to <8 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f16_v16f32 = fpext <16 x half> undef to <16 x float> +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16f16_v16f64 = fpext <16 x half> undef to <16 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16f32_v16f64 = fpext <16 x float> undef to <16 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v32f16_v32f32 = fpext <32 x half> undef to <32 x float> +; CHECK-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %v32f16_v32f64 = fpext <32 x half> undef to <32 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v32f32_v32f64 = fpext <32 x float> undef to <32 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v64f16_v64f32 = fpext <64 x half> undef to <64 x float> +; CHECK-NEXT: Cost Model: Found an estimated cost of 51 for instruction: %v64f16_v64f64 = fpext <64 x half> undef to <64 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v64f32_v64f64 = fpext <64 x float> undef to <64 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v128f16_v128f32 = fpext <128 x half> undef to <128 x float> +; CHECK-NEXT: Cost Model: Found an estimated cost of 102 for instruction: %v128f16_v128f64 = fpext <128 x half> undef to <128 x double> +; CHECK-NEXT: Cost Model: Found an estimated cost of 68 for instruction: %v128f32_v128f64 = fpext <128 x float> undef to <128 x double> ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16_nxv1f32 = fpext undef to ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f16_nxv1f64 = fpext undef to ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1f64 = fpext undef to ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16_nxv2f32 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f16_nxv2f64 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32_nxv2f64 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16_nxv4f32 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16_nxv4f64 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32_nxv4f64 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16_nxv8f32 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16_nxv8f64 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32_nxv8f64 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16_nxv16f32 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16f16_nxv16f64 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv16f32_nxv16f64 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv32f16_nxv32f32 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv32f16_nxv32f64 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv32f32_nxv32f64 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv64f16_nxv64f32 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %nxv64f16_nxv64f64 = fpext undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %nxv64f32_nxv64f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f16_nxv2f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32_nxv2f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16_nxv4f32 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv4f16_nxv4f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f32_nxv4f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f16_nxv8f32 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %nxv8f16_nxv8f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f32_nxv8f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f16_nxv16f32 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %nxv16f16_nxv16f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv16f32_nxv16f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv32f16_nxv32f32 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 51 for instruction: %nxv32f16_nxv32f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %nxv32f32_nxv32f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %nxv64f16_nxv64f32 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 102 for instruction: %nxv64f16_nxv64f64 = fpext undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 68 for instruction: %nxv64f32_nxv64f64 = fpext undef to ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16_v2f32 = fpext <2 x half> undef to <2 x float> @@ -1603,20 +1603,20 @@ define void @fptrunc() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64_v4f16 = fptrunc <4 x double> undef to <4 x half> ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64_v4f32 = fptrunc <4 x double> undef to <4 x float> ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32_v8f16 = fptrunc <8 x float> undef to <8 x half> -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f64_v8f16 = fptrunc <8 x double> undef to <8 x half> -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64_v8f32 = fptrunc <8 x double> undef to <8 x float> -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32_v16f16 = fptrunc <16 x float> undef to <16 x half> -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f64_v16f16 = fptrunc <16 x double> undef to <16 x half> -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f64_v16f32 = fptrunc <16 x double> undef to <16 x float> -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32f32_v32f16 = fptrunc <32 x float> undef to <32 x half> -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v32f64_v32f16 = fptrunc <32 x double> undef to <32 x half> -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v32f64_v32f32 = fptrunc <32 x double> undef to <32 x float> -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v64f32_v64f16 = fptrunc <64 x float> undef to <64 x half> -; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v64f64_v64f16 = fptrunc <64 x double> undef to <64 x half> -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v64f64_v64f32 = fptrunc <64 x double> undef to <64 x float> -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v128f32_v128f16 = fptrunc <128 x float> undef to <128 x half> -; CHECK-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v128f64_v128f16 = fptrunc <128 x double> undef to <128 x half> -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v128f64_v128f32 = fptrunc <128 x double> undef to <128 x float> +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f64_v8f16 = fptrunc <8 x double> undef to <8 x half> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f64_v8f32 = fptrunc <8 x double> undef to <8 x float> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f32_v16f16 = fptrunc <16 x float> undef to <16 x half> +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v16f64_v16f16 = fptrunc <16 x double> undef to <16 x half> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f64_v16f32 = fptrunc <16 x double> undef to <16 x float> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32f32_v32f16 = fptrunc <32 x float> undef to <32 x half> +; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %v32f64_v32f16 = fptrunc <32 x double> undef to <32 x half> +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v32f64_v32f32 = fptrunc <32 x double> undef to <32 x float> +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v64f32_v64f16 = fptrunc <64 x float> undef to <64 x half> +; CHECK-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %v64f64_v64f16 = fptrunc <64 x double> undef to <64 x half> +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v64f64_v64f32 = fptrunc <64 x double> undef to <64 x float> +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v128f32_v128f16 = fptrunc <128 x float> undef to <128 x half> +; CHECK-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %v128f64_v128f16 = fptrunc <128 x double> undef to <128 x half> +; CHECK-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v128f64_v128f32 = fptrunc <128 x double> undef to <128 x float> ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1f16 = fptrunc undef to ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64_nxv1f16 = fptrunc undef to ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64_nxv1f32 = fptrunc undef to @@ -1624,20 +1624,20 @@ define void @fptrunc() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64_nxv1f16 = fptrunc undef to ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64_nxv1f32 = fptrunc undef to ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32_nxv4f16 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f64_nxv4f16 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64_nxv4f32 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32_nxv8f16 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f64_nxv8f16 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64_nxv8f32 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32_nxv16f16 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16f64_nxv16f16 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv16f64_nxv16f32 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv32f32_nxv32f16 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv32f64_nxv32f16 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv32f64_nxv32f32 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv64f32_nxv64f16 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %nxv64f64_nxv64f16 = fptrunc undef to -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %nxv64f64_nxv64f32 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f64_nxv4f16 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f64_nxv4f32 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f32_nxv8f16 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv8f64_nxv8f16 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f64_nxv8f32 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f32_nxv16f16 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %nxv16f64_nxv16f16 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f64_nxv16f32 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv32f32_nxv32f16 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %nxv32f64_nxv32f16 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %nxv32f64_nxv32f32 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %nxv64f32_nxv64f16 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %nxv64f64_nxv64f16 = fptrunc undef to +; CHECK-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %nxv64f64_nxv64f32 = fptrunc undef to ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f32_v2f16 = fptrunc <2 x float> undef to <2 x half> diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll index e42dc889f1ba..a9a5f4d2de54 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll @@ -238,7 +238,7 @@ define float @vreduce_ord_fadd_nxv4f32( %v, float %s) { define float @vreduce_fwadd_nxv4f32( %v, float %s) { ; CHECK-LABEL: 'vreduce_fwadd_nxv4f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %e = fpext %v to +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %e = fpext %v to ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %red = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float %s, %e) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; @@ -254,7 +254,7 @@ define float @vreduce_fwadd_nxv4f32( %v, float %s) { define float @vreduce_ord_fwadd_nxv4f32( %v, float %s) { ; CHECK-LABEL: 'vreduce_ord_fwadd_nxv4f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %e = fpext %v to +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %e = fpext %v to ; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %red = call float @llvm.vector.reduce.fadd.nxv4f32(float %s, %e) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float %red ; @@ -358,7 +358,7 @@ define double @vreduce_ord_fadd_nxv2f64( %v, double %s) { define double @vreduce_fwadd_nxv2f64( %v, double %s) { ; CHECK-LABEL: 'vreduce_fwadd_nxv2f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %e = fpext %v to +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %e = fpext %v to ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %red = call reassoc double @llvm.vector.reduce.fadd.nxv2f64(double %s, %e) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; @@ -374,7 +374,7 @@ define double @vreduce_fwadd_nxv2f64( %v, double %s) { define double @vreduce_ord_fwadd_nxv2f64( %v, double %s) { ; CHECK-LABEL: 'vreduce_ord_fwadd_nxv2f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %e = fpext %v to +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %e = fpext %v to ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %red = call double @llvm.vector.reduce.fadd.nxv2f64(double %s, %e) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; @@ -418,7 +418,7 @@ define double @vreduce_ord_fadd_nxv4f64( %v, double %s) { define double @vreduce_fwadd_nxv4f64( %v, double %s) { ; CHECK-LABEL: 'vreduce_fwadd_nxv4f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %e = fpext %v to +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %e = fpext %v to ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %red = call reassoc double @llvm.vector.reduce.fadd.nxv4f64(double %s, %e) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; @@ -434,7 +434,7 @@ define double @vreduce_fwadd_nxv4f64( %v, double %s) { define double @vreduce_ord_fwadd_nxv4f64( %v, double %s) { ; CHECK-LABEL: 'vreduce_ord_fwadd_nxv4f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %e = fpext %v to +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %e = fpext %v to ; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %red = call double @llvm.vector.reduce.fadd.nxv4f64(double %s, %e) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double %red ; diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll index 6e1ae0216f76..8b68480788f7 100644 --- a/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll +++ b/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll @@ -12,12 +12,12 @@ define void @insertelement_int(i32 %x) { ; RV32V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0 ; RV32V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0 ; RV32V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0 -; RV32V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0 +; RV32V-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0 ; RV32V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement undef, i1 undef, i32 0 ; RV32V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement undef, i1 undef, i32 0 ; RV32V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement undef, i1 undef, i32 0 -; RV32V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement undef, i1 undef, i32 0 -; RV32V-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement undef, i1 undef, i32 0 +; RV32V-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_0 = insertelement undef, i1 undef, i32 0 +; RV32V-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv32i1_0 = insertelement undef, i1 undef, i32 0 ; RV32V-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0 ; RV32V-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0 ; RV32V-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0 @@ -66,12 +66,12 @@ define void @insertelement_int(i32 %x) { ; RV32V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1 ; RV32V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1 ; RV32V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1 -; RV32V-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1 +; RV32V-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1 ; RV32V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement undef, i1 undef, i32 1 ; RV32V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement undef, i1 undef, i32 1 ; RV32V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement undef, i1 undef, i32 1 -; RV32V-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement undef, i1 undef, i32 1 -; RV32V-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement undef, i1 undef, i32 1 +; RV32V-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %nxv16i1_1 = insertelement undef, i1 undef, i32 1 +; RV32V-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %nxv32i1_1 = insertelement undef, i1 undef, i32 1 ; RV32V-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1 ; RV32V-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1 ; RV32V-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1 @@ -120,12 +120,12 @@ define void @insertelement_int(i32 %x) { ; RV32V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x ; RV32V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x ; RV32V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x -; RV32V-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x +; RV32V-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x ; RV32V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement undef, i1 undef, i32 %x ; RV32V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement undef, i1 undef, i32 %x ; RV32V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement undef, i1 undef, i32 %x -; RV32V-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement undef, i1 undef, i32 %x -; RV32V-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement undef, i1 undef, i32 %x +; RV32V-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv16i1_x = insertelement undef, i1 undef, i32 %x +; RV32V-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %nxv32i1_x = insertelement undef, i1 undef, i32 %x ; RV32V-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x ; RV32V-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x ; RV32V-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x @@ -177,12 +177,12 @@ define void @insertelement_int(i32 %x) { ; RV64V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0 ; RV64V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0 ; RV64V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0 -; RV64V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0 +; RV64V-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0 ; RV64V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement undef, i1 undef, i32 0 ; RV64V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement undef, i1 undef, i32 0 ; RV64V-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement undef, i1 undef, i32 0 -; RV64V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement undef, i1 undef, i32 0 -; RV64V-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement undef, i1 undef, i32 0 +; RV64V-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_0 = insertelement undef, i1 undef, i32 0 +; RV64V-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv32i1_0 = insertelement undef, i1 undef, i32 0 ; RV64V-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0 ; RV64V-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0 ; RV64V-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0 @@ -231,12 +231,12 @@ define void @insertelement_int(i32 %x) { ; RV64V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1 ; RV64V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1 ; RV64V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1 -; RV64V-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1 +; RV64V-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1 ; RV64V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement undef, i1 undef, i32 1 ; RV64V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement undef, i1 undef, i32 1 ; RV64V-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement undef, i1 undef, i32 1 -; RV64V-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement undef, i1 undef, i32 1 -; RV64V-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement undef, i1 undef, i32 1 +; RV64V-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %nxv16i1_1 = insertelement undef, i1 undef, i32 1 +; RV64V-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %nxv32i1_1 = insertelement undef, i1 undef, i32 1 ; RV64V-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1 ; RV64V-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1 ; RV64V-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1 @@ -285,12 +285,12 @@ define void @insertelement_int(i32 %x) { ; RV64V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x ; RV64V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x ; RV64V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x -; RV64V-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x +; RV64V-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x ; RV64V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement undef, i1 undef, i32 %x ; RV64V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement undef, i1 undef, i32 %x ; RV64V-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement undef, i1 undef, i32 %x -; RV64V-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement undef, i1 undef, i32 %x -; RV64V-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement undef, i1 undef, i32 %x +; RV64V-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv16i1_x = insertelement undef, i1 undef, i32 %x +; RV64V-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %nxv32i1_x = insertelement undef, i1 undef, i32 %x ; RV64V-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x ; RV64V-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x ; RV64V-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x @@ -341,13 +341,13 @@ define void @insertelement_int(i32 %x) { ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v2i1_0 = insertelement <2 x i1> undef, i1 undef, i32 0 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0 -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0 -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0 +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0 +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement undef, i1 undef, i32 0 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement undef, i1 undef, i32 0 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement undef, i1 undef, i32 0 -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement undef, i1 undef, i32 0 -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement undef, i1 undef, i32 0 +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_0 = insertelement undef, i1 undef, i32 0 +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv32i1_0 = insertelement undef, i1 undef, i32 0 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0 @@ -395,13 +395,13 @@ define void @insertelement_int(i32 %x) { ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2i1_1 = insertelement <2 x i1> undef, i1 undef, i32 1 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1 -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1 -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1 +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1 +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement undef, i1 undef, i32 1 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement undef, i1 undef, i32 1 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement undef, i1 undef, i32 1 -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement undef, i1 undef, i32 1 -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement undef, i1 undef, i32 1 +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %nxv16i1_1 = insertelement undef, i1 undef, i32 1 +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %nxv32i1_1 = insertelement undef, i1 undef, i32 1 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1 ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1 @@ -449,13 +449,13 @@ define void @insertelement_int(i32 %x) { ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v2i1_x = insertelement <2 x i1> undef, i1 undef, i32 %x ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement undef, i1 undef, i32 %x ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement undef, i1 undef, i32 %x ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement undef, i1 undef, i32 %x -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement undef, i1 undef, i32 %x -; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement undef, i1 undef, i32 %x +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv16i1_x = insertelement undef, i1 undef, i32 %x +; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %nxv32i1_x = insertelement undef, i1 undef, i32 %x ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x ; RV32ZVE64X-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x @@ -506,13 +506,13 @@ define void @insertelement_int(i32 %x) { ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v2i1_0 = insertelement <2 x i1> undef, i1 undef, i32 0 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0 -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0 -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0 +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0 +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement undef, i1 undef, i32 0 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement undef, i1 undef, i32 0 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement undef, i1 undef, i32 0 -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement undef, i1 undef, i32 0 -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement undef, i1 undef, i32 0 +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_0 = insertelement undef, i1 undef, i32 0 +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv32i1_0 = insertelement undef, i1 undef, i32 0 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0 @@ -560,13 +560,13 @@ define void @insertelement_int(i32 %x) { ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2i1_1 = insertelement <2 x i1> undef, i1 undef, i32 1 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1 -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1 -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1 +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1 +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement undef, i1 undef, i32 1 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement undef, i1 undef, i32 1 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement undef, i1 undef, i32 1 -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement undef, i1 undef, i32 1 -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement undef, i1 undef, i32 1 +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %nxv16i1_1 = insertelement undef, i1 undef, i32 1 +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %nxv32i1_1 = insertelement undef, i1 undef, i32 1 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1 ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1 @@ -614,13 +614,13 @@ define void @insertelement_int(i32 %x) { ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v2i1_x = insertelement <2 x i1> undef, i1 undef, i32 %x ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement undef, i1 undef, i32 %x ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement undef, i1 undef, i32 %x ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement undef, i1 undef, i32 %x -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement undef, i1 undef, i32 %x -; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement undef, i1 undef, i32 %x +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %nxv16i1_x = insertelement undef, i1 undef, i32 %x +; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %nxv32i1_x = insertelement undef, i1 undef, i32 %x ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x ; RV64ZVE64X-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll index b763198e98ba..79ba1562d0f8 100644 --- a/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll +++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll @@ -197,7 +197,7 @@ define void @broadcast_fixed() #0{ ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %41 = shufflevector <32 x i1> undef, <32 x i1> undef, <32 x i32> zeroinitializer ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %42 = shufflevector <64 x i1> undef, <64 x i1> undef, <64 x i32> zeroinitializer ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %43 = shufflevector <128 x i1> undef, <128 x i1> undef, <128 x i32> zeroinitializer -; CHECK-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %ins1 = insertelement <128 x i1> poison, i1 poison, i32 0 +; CHECK-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %ins1 = insertelement <128 x i1> poison, i1 poison, i32 0 ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %44 = shufflevector <128 x i1> %ins1, <128 x i1> poison, <128 x i32> zeroinitializer ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ins2 = insertelement <2 x i8> poison, i8 3, i32 0 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %45 = shufflevector <2 x i8> %ins2, <2 x i8> undef, <2 x i32> zeroinitializer -- GitLab From 38113a083283d2f30a677befaa5fb86dce731c8b Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Tue, 2 Apr 2024 10:53:57 +0900 Subject: [PATCH 0158/1690] [mlir][IR] Trigger `notifyOperationReplaced` on `replaceAllOpUsesWith` (#84721) Before this change: `notifyOperationReplaced` was triggered when calling `RewriteBase::replaceOp`. After this change: `notifyOperationReplaced` is triggered when `RewriterBase::replaceAllOpUsesWith` or `RewriterBase::replaceOp` is called. Until now, every `notifyOperationReplaced` was always sent together with a `notifyOperationErased`, which made that `notifyOperationErased` callback irrelevant. More importantly, when a user called `RewriterBase::replaceAllOpUsesWith`+`RewriterBase::eraseOp` instead of `RewriterBase::replaceOp`, no `notifyOperationReplaced` callback was sent, even though the two notations are semantically equivalent. As an example, this can be a problem when applying patterns with the transform dialect because the `TrackingListener` will only see the `notifyOperationErased` callback and the payload op is dropped from the mappings. Note: It is still possible to write semantically equivalent code that does not trigger a `notifyOperationReplaced` (e.g., when op results are replaced one-by-one), but this commit already improves the situation a lot. --- mlir/include/mlir/IR/PatternMatch.h | 29 ++++++++++++--------- mlir/lib/IR/PatternMatch.cpp | 24 +++++++++++------ mlir/test/lib/Dialect/Test/TestPatterns.cpp | 5 +++- 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h index 070e6ed702f8..ac2b0d5a3837 100644 --- a/mlir/include/mlir/IR/PatternMatch.h +++ b/mlir/include/mlir/IR/PatternMatch.h @@ -409,9 +409,9 @@ public: /// Notify the listener that the specified operation was modified in-place. virtual void notifyOperationModified(Operation *op) {} - /// Notify the listener that the specified operation is about to be replaced - /// with another operation. This is called before the uses of the old - /// operation have been changed. + /// Notify the listener that all uses of the specified operation's results + /// are about to be replaced with the results of another operation. This is + /// called before the uses of the old operation have been changed. /// /// By default, this function calls the "operation replaced with values" /// notification. @@ -420,9 +420,10 @@ public: notifyOperationReplaced(op, replacement->getResults()); } - /// Notify the listener that the specified operation is about to be replaced - /// with the a range of values, potentially produced by other operations. - /// This is called before the uses of the operation have been changed. + /// Notify the listener that all uses of the specified operation's results + /// are about to be replaced with the a range of values, potentially + /// produced by other operations. This is called before the uses of the + /// operation have been changed. virtual void notifyOperationReplaced(Operation *op, ValueRange replacement) {} @@ -648,12 +649,16 @@ public: for (auto it : llvm::zip(from, to)) replaceAllUsesWith(std::get<0>(it), std::get<1>(it)); } - // Note: This function cannot be called `replaceAllUsesWith` because the - // overload resolution, when called with an op that can be implicitly - // converted to a Value, would be ambiguous. - void replaceAllOpUsesWith(Operation *from, ValueRange to) { - replaceAllUsesWith(from->getResults(), to); - } + + /// Find uses of `from` and replace them with `to`. Also notify the listener + /// about every in-place op modification (for every use that was replaced) + /// and that the `from` operation is about to be replaced. + /// + /// Note: This function cannot be called `replaceAllUsesWith` because the + /// overload resolution, when called with an op that can be implicitly + /// converted to a Value, would be ambiguous. + void replaceAllOpUsesWith(Operation *from, ValueRange to); + void replaceAllOpUsesWith(Operation *from, Operation *to); /// Find uses of `from` and replace them with `to` if the `functor` returns /// true. Also notify the listener about every in-place op modification (for diff --git a/mlir/lib/IR/PatternMatch.cpp b/mlir/lib/IR/PatternMatch.cpp index 4079ccc75672..5944a0ea46a1 100644 --- a/mlir/lib/IR/PatternMatch.cpp +++ b/mlir/lib/IR/PatternMatch.cpp @@ -110,6 +110,22 @@ RewriterBase::~RewriterBase() { // Out of line to provide a vtable anchor for the class. } +void RewriterBase::replaceAllOpUsesWith(Operation *from, ValueRange to) { + // Notify the listener that we're about to replace this op. + if (auto *rewriteListener = dyn_cast_if_present(listener)) + rewriteListener->notifyOperationReplaced(from, to); + + replaceAllUsesWith(from->getResults(), to); +} + +void RewriterBase::replaceAllOpUsesWith(Operation *from, Operation *to) { + // Notify the listener that we're about to replace this op. + if (auto *rewriteListener = dyn_cast_if_present(listener)) + rewriteListener->notifyOperationReplaced(from, to); + + replaceAllUsesWith(from->getResults(), to->getResults()); +} + /// This method replaces the results of the operation with the specified list of /// values. The number of provided values must match the number of results of /// the operation. The replaced op is erased. @@ -117,10 +133,6 @@ void RewriterBase::replaceOp(Operation *op, ValueRange newValues) { assert(op->getNumResults() == newValues.size() && "incorrect # of replacement values"); - // Notify the listener that we're about to replace this op. - if (auto *rewriteListener = dyn_cast_if_present(listener)) - rewriteListener->notifyOperationReplaced(op, newValues); - // Replace all result uses. Also notifies the listener of modifications. replaceAllOpUsesWith(op, newValues); @@ -136,10 +148,6 @@ void RewriterBase::replaceOp(Operation *op, Operation *newOp) { assert(op->getNumResults() == newOp->getNumResults() && "ops have different number of results"); - // Notify the listener that we're about to replace this op. - if (auto *rewriteListener = dyn_cast_if_present(listener)) - rewriteListener->notifyOperationReplaced(op, newOp); - // Replace all result uses. Also notifies the listener of modifications. replaceAllOpUsesWith(op, newOp->getResults()); diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp index 2da184bc3d85..76dc825fe445 100644 --- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp +++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp @@ -489,7 +489,10 @@ private: OperationName("test.new_op", op->getContext()).getIdentifier(), op->getOperands(), op->getResultTypes()); } - rewriter.replaceOp(op, newOp->getResults()); + // "replaceOp" could be used instead of "replaceAllOpUsesWith"+"eraseOp". + // A "notifyOperationReplaced" callback is triggered in either case. + rewriter.replaceAllOpUsesWith(op, newOp->getResults()); + rewriter.eraseOp(op); return success(); } }; -- GitLab From 21f85e230056172cffcaec76352e5a2019b54b86 Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Tue, 2 Apr 2024 09:52:52 +0800 Subject: [PATCH 0159/1690] [NFC] [C++20] [Modules] Pulling out getCXX20NamedModuleOutputPath into a seperate function Required in the review process of https://github.com/llvm/llvm-project/pull/85050. --- clang/lib/Driver/Driver.cpp | 14 ++------------ clang/lib/Driver/ToolChains/Clang.cpp | 18 ++++++++++++++++++ clang/lib/Driver/ToolChains/Clang.h | 15 +++++++++++++++ 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index 7a53764364ce..1a0f5f27eda2 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -5814,19 +5814,9 @@ static const char *GetModuleOutputPath(Compilation &C, const JobAction &JA, (C.getArgs().hasArg(options::OPT_fmodule_output) || C.getArgs().hasArg(options::OPT_fmodule_output_EQ))); - if (Arg *ModuleOutputEQ = - C.getArgs().getLastArg(options::OPT_fmodule_output_EQ)) - return C.addResultFile(ModuleOutputEQ->getValue(), &JA); + SmallString<256> OutputPath = + tools::getCXX20NamedModuleOutputPath(C.getArgs(), BaseInput); - SmallString<64> OutputPath; - Arg *FinalOutput = C.getArgs().getLastArg(options::OPT_o); - if (FinalOutput && C.getArgs().hasArg(options::OPT_c)) - OutputPath = FinalOutput->getValue(); - else - OutputPath = BaseInput; - - const char *Extension = types::getTypeTempSuffix(JA.getType()); - llvm::sys::path::replace_extension(OutputPath, Extension); return C.addResultFile(C.getArgs().MakeArgString(OutputPath.c_str()), &JA); } diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 3bcacff7724c..b03ac6018d2b 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -3839,6 +3839,24 @@ bool Driver::getDefaultModuleCachePath(SmallVectorImpl &Result) { return false; } +llvm::SmallString<256> +clang::driver::tools::getCXX20NamedModuleOutputPath(const ArgList &Args, + const char *BaseInput) { + if (Arg *ModuleOutputEQ = Args.getLastArg(options::OPT_fmodule_output_EQ)) + return StringRef(ModuleOutputEQ->getValue()); + + SmallString<256> OutputPath; + if (Arg *FinalOutput = Args.getLastArg(options::OPT_o); + FinalOutput && Args.hasArg(options::OPT_c)) + OutputPath = FinalOutput->getValue(); + else + OutputPath = BaseInput; + + const char *Extension = types::getTypeTempSuffix(types::TY_ModuleFile); + llvm::sys::path::replace_extension(OutputPath, Extension); + return OutputPath; +} + static bool RenderModulesOptions(Compilation &C, const Driver &D, const ArgList &Args, const InputInfo &Input, const InputInfo &Output, bool HaveStd20, diff --git a/clang/lib/Driver/ToolChains/Clang.h b/clang/lib/Driver/ToolChains/Clang.h index 0f503c4bd1c4..18f6c5ed06a5 100644 --- a/clang/lib/Driver/ToolChains/Clang.h +++ b/clang/lib/Driver/ToolChains/Clang.h @@ -193,6 +193,21 @@ DwarfFissionKind getDebugFissionKind(const Driver &D, const llvm::opt::ArgList &Args, llvm::opt::Arg *&Arg); +// Calculate the output path of the module file when compiling a module unit +// with the `-fmodule-output` option or `-fmodule-output=` option specified. +// The behavior is: +// - If `-fmodule-output=` is specfied, then the module file is +// writing to the value. +// - Otherwise if the output object file of the module unit is specified, the +// output path +// of the module file should be the same with the output object file except +// the corresponding suffix. This requires both `-o` and `-c` are specified. +// - Otherwise, the output path of the module file will be the same with the +// input with the corresponding suffix. +llvm::SmallString<256> +getCXX20NamedModuleOutputPath(const llvm::opt::ArgList &Args, + const char *BaseInput); + } // end namespace tools } // end namespace driver -- GitLab From 9067f5470573454ad33f2d1786cdfa77f7f9329c Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Tue, 2 Apr 2024 11:03:12 +0900 Subject: [PATCH 0160/1690] [mlir][IR][NFC] Make `replaceAllUsesWith` non-templatized (#84722) Turn `RewriterBase::replaceAllUsesWith` into a non-templatized implementation, so that it can be made virtual and be overridden in the `ConversionPatternRewriter` in a subsequent change. This change is in preparation of adding dialect conversion support for `replaceAllUsesWith`. --- mlir/include/mlir/IR/PatternMatch.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h index ac2b0d5a3837..15b1c3892948 100644 --- a/mlir/include/mlir/IR/PatternMatch.h +++ b/mlir/include/mlir/IR/PatternMatch.h @@ -635,11 +635,13 @@ public: /// Find uses of `from` and replace them with `to`. Also notify the listener /// about every in-place op modification (for every use that was replaced). void replaceAllUsesWith(Value from, Value to) { - return replaceAllUsesWith(from.getImpl(), to); + for (OpOperand &operand : llvm::make_early_inc_range(from.getUses())) { + Operation *op = operand.getOwner(); + modifyOpInPlace(op, [&]() { operand.set(to); }); + } } - template - void replaceAllUsesWith(IRObjectWithUseList *from, ValueT &&to) { - for (OperandType &operand : llvm::make_early_inc_range(from->getUses())) { + void replaceAllUsesWith(Block *from, Block *to) { + for (BlockOperand &operand : llvm::make_early_inc_range(from->getUses())) { Operation *op = operand.getOwner(); modifyOpInPlace(op, [&]() { operand.set(to); }); } -- GitLab From 49a4ec20a8be5888cbf225bab340dbaf204902c7 Mon Sep 17 00:00:00 2001 From: Rob Suderman Date: Mon, 1 Apr 2024 19:22:49 -0700 Subject: [PATCH 0161/1690] [mlir] Reland the dialect conversion hanging use fix (#87297) Dialect conversion sometimes can have a hanging use of an argument. Ensured that argument uses are dropped before removing the block. --- mlir/lib/Transforms/Utils/DialectConversion.cpp | 2 ++ .../TosaToLinalg/tosa-to-linalg-invalid.mlir | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index 8671c1008902..270ac0a08689 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -279,6 +279,8 @@ public: auto &blockOps = block->getOperations(); while (!blockOps.empty()) blockOps.remove(blockOps.begin()); + for (auto arg : block->getArguments()) + arg.dropAllUses(); block->dropAllUses(); if (block->getParent()) block->erase(); diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir index 17eec5936918..6494e1b27194 100644 --- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir +++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir @@ -15,3 +15,16 @@ func.func @tensor_with_unknown_rank(%arg0: tensor<*xi8>) -> tensor<*xi8> { %0 = "tosa.abs"(%arg0) : (tensor<*xi8>) -> tensor<*xi8> return %0 : tensor<*xi8> } + +// ----- + +// CHECK-LABEL: @unranked_add +func.func @unranked_add(%arg0 : tensor<10x10xf32> , %arg1 : tensor<10x10xf32>, %arg2 : tensor<*xf32>) -> (tensor<10x10xf32>) { + // expected-error@+3 {{failed to legalize operation 'tosa.add'}} + %reduce = tosa.reduce_max %arg0 {axis = 1 : i32} : (tensor<10x10xf32>) -> tensor<10x1xf32> + %1 = tosa.add %reduce, %arg1 : (tensor<10x1xf32>, tensor<10x10xf32>) -> tensor<10x10xf32> + %0 = tosa.add %1, %arg2 : (tensor<10x10xf32>, tensor<*xf32>) -> tensor<*xf32> + %2 = tosa.reshape %0 {new_shape = array} : (tensor<*xf32>) -> tensor<10x10xf32> + return %2 : tensor<10x10xf32> +} + -- GitLab From b932db08bb8e56c80380468698a6f75d5ea35577 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Mon, 1 Apr 2024 20:19:59 -0700 Subject: [PATCH 0162/1690] [llvm-objcopy,test] Prepend error: to some messages --- llvm/test/tools/llvm-objcopy/ELF/discard-locals-rel.test | 4 ++-- llvm/test/tools/llvm-objcopy/ELF/strip-reloc-symbol.test | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/test/tools/llvm-objcopy/ELF/discard-locals-rel.test b/llvm/test/tools/llvm-objcopy/ELF/discard-locals-rel.test index 3658eb376010..00bb8fcf1820 100644 --- a/llvm/test/tools/llvm-objcopy/ELF/discard-locals-rel.test +++ b/llvm/test/tools/llvm-objcopy/ELF/discard-locals-rel.test @@ -1,5 +1,5 @@ # RUN: yaml2obj %s -o %t -# RUN: not llvm-objcopy --discard-locals %t %t2 2>&1 | FileCheck %s +# RUN: not llvm-objcopy --discard-locals %t %t2 2>&1 | FileCheck %s -DFILE=%t !ELF FileHeader: @@ -23,4 +23,4 @@ Symbols: Type: STT_FUNC Section: .text -# CHECK: not stripping symbol '.L.rel' because it is named in a relocation +# CHECK: error: '[[FILE]]': not stripping symbol '.L.rel' because it is named in a relocation diff --git a/llvm/test/tools/llvm-objcopy/ELF/strip-reloc-symbol.test b/llvm/test/tools/llvm-objcopy/ELF/strip-reloc-symbol.test index 63c9e122d9a2..941dacce2edf 100644 --- a/llvm/test/tools/llvm-objcopy/ELF/strip-reloc-symbol.test +++ b/llvm/test/tools/llvm-objcopy/ELF/strip-reloc-symbol.test @@ -1,5 +1,5 @@ # RUN: yaml2obj %s -o %t -# RUN: not llvm-objcopy -N foo %t %t2 2>&1 | FileCheck %s +# RUN: not llvm-objcopy -N foo %t %t2 2>&1 | FileCheck %s -DFILE=%t !ELF FileHeader: @@ -28,4 +28,4 @@ Symbols: Value: 0x1000 Size: 8 -# CHECK: not stripping symbol 'foo' because it is named in a relocation +# CHECK: error: '[[FILE]]': not stripping symbol 'foo' because it is named in a relocation -- GitLab From 59dd10faf8c3bb9dbcecb60d932284b8762cebf8 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Tue, 2 Apr 2024 13:02:03 +0800 Subject: [PATCH 0163/1690] [RISCV] Add tests for fixed vector vwsll. NFC We are missing patterns for fixed vectors, where the sexts and zexts are legalized to _vl nodes. --- .../CodeGen/RISCV/rvv/fixed-vectors-vwsll.ll | 920 ++++++++++++++++++ 1 file changed, 920 insertions(+) create mode 100644 llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsll.ll diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsll.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsll.ll new file mode 100644 index 000000000000..f5305a1c36de --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsll.ll @@ -0,0 +1,920 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 +; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=riscv32 -mattr=+v,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK-ZVBB +; RUN: llc -mtriple=riscv64 -mattr=+v,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK-ZVBB + +; ============================================================================== +; i32 -> i64 +; ============================================================================== + +define <4 x i64> @vwsll_vv_v4i64_sext(<4 x i32> %a, <4 x i32> %b) { +; CHECK-LABEL: vwsll_vv_v4i64_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vv_v4i64_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %x = zext <4 x i32> %a to <4 x i64> + %y = sext <4 x i32> %b to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vv_v4i64_zext(<4 x i32> %a, <4 x i32> %b) { +; CHECK-LABEL: vwsll_vv_v4i64_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vv_v4i64_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %x = zext <4 x i32> %a to <4 x i64> + %y = zext <4 x i32> %b to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i64_v4i64(<4 x i32> %a, i64 %b) { +; CHECK-LABEL: vwsll_vx_i64_v4i64: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsll.vx v8, v10, a0 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i64_v4i64: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsll.vx v8, v10, a0 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i64> poison, i64 %b, i32 0 + %splat = shufflevector <4 x i64> %head, <4 x i64> poison, <4 x i32> zeroinitializer + %x = zext <4 x i32> %a to <4 x i64> + %z = shl <4 x i64> %x, %splat + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i32_v4i64_sext(<4 x i32> %a, i32 %b) { +; CHECK-LABEL: vwsll_vx_i32_v4i64_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i32_v4i64_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i32> poison, i32 %b, i32 0 + %splat = shufflevector <4 x i32> %head, <4 x i32> poison, <4 x i32> zeroinitializer + %x = zext <4 x i32> %a to <4 x i64> + %y = sext <4 x i32> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i32_v4i64_zext(<4 x i32> %a, i32 %b) { +; CHECK-LABEL: vwsll_vx_i32_v4i64_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i32_v4i64_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i32> poison, i32 %b, i32 0 + %splat = shufflevector <4 x i32> %head, <4 x i32> poison, <4 x i32> zeroinitializer + %x = zext <4 x i32> %a to <4 x i64> + %y = zext <4 x i32> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i16_v4i64_sext(<4 x i32> %a, i16 %b) { +; CHECK-LABEL: vwsll_vx_i16_v4i64_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf4 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i16_v4i64_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf4 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i16> poison, i16 %b, i32 0 + %splat = shufflevector <4 x i16> %head, <4 x i16> poison, <4 x i32> zeroinitializer + %x = zext <4 x i32> %a to <4 x i64> + %y = sext <4 x i16> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i16_v4i64_zext(<4 x i32> %a, i16 %b) { +; CHECK-LABEL: vwsll_vx_i16_v4i64_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf4 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i16_v4i64_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf4 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i16> poison, i16 %b, i32 0 + %splat = shufflevector <4 x i16> %head, <4 x i16> poison, <4 x i32> zeroinitializer + %x = zext <4 x i32> %a to <4 x i64> + %y = zext <4 x i16> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i8_v4i64_sext(<4 x i32> %a, i8 %b) { +; CHECK-LABEL: vwsll_vx_i8_v4i64_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf8 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i8_v4i64_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf8 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i8> poison, i8 %b, i32 0 + %splat = shufflevector <4 x i8> %head, <4 x i8> poison, <4 x i32> zeroinitializer + %x = zext <4 x i32> %a to <4 x i64> + %y = sext <4 x i8> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i8_v4i64_zext(<4 x i32> %a, i8 %b) { +; CHECK-LABEL: vwsll_vx_i8_v4i64_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf8 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i8_v4i64_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf8 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i8> poison, i8 %b, i32 0 + %splat = shufflevector <4 x i8> %head, <4 x i8> poison, <4 x i32> zeroinitializer + %x = zext <4 x i32> %a to <4 x i64> + %y = zext <4 x i8> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vi_v4i64(<4 x i32> %a) { +; CHECK-LABEL: vwsll_vi_v4i64: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsll.vi v8, v10, 2 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vi_v4i64: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsll.vi v8, v10, 2 +; CHECK-ZVBB-NEXT: ret + %x = zext <4 x i32> %a to <4 x i64> + %z = shl <4 x i64> %x, splat (i64 2) + ret <4 x i64> %z +} + +; ============================================================================== +; i16 -> i32 +; ============================================================================== + +define <8 x i32> @vwsll_vv_v8i32_sext(<8 x i16> %a, <8 x i16> %b) { +; CHECK-LABEL: vwsll_vv_v8i32_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vv_v8i32_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %x = zext <8 x i16> %a to <8 x i32> + %y = sext <8 x i16> %b to <8 x i32> + %z = shl <8 x i32> %x, %y + ret <8 x i32> %z +} + +define <8 x i32> @vwsll_vv_v8i32_zext(<8 x i16> %a, <8 x i16> %b) { +; CHECK-LABEL: vwsll_vv_v8i32_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vv_v8i32_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %x = zext <8 x i16> %a to <8 x i32> + %y = zext <8 x i16> %b to <8 x i32> + %z = shl <8 x i32> %x, %y + ret <8 x i32> %z +} + +define <8 x i32> @vwsll_vx_i64_v8i32(<8 x i16> %a, i64 %b) { +; CHECK-LABEL: vwsll_vx_i64_v8i32: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsll.vx v8, v10, a0 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i64_v8i32: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsll.vx v8, v10, a0 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <8 x i64> poison, i64 %b, i32 0 + %splat = shufflevector <8 x i64> %head, <8 x i64> poison, <8 x i32> zeroinitializer + %x = zext <8 x i16> %a to <8 x i32> + %y = trunc <8 x i64> %splat to <8 x i32> + %z = shl <8 x i32> %x, %y + ret <8 x i32> %z +} + +define <8 x i32> @vwsll_vx_i32_v8i32(<8 x i16> %a, i32 %b) { +; CHECK-LABEL: vwsll_vx_i32_v8i32: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsll.vx v8, v10, a0 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i32_v8i32: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsll.vx v8, v10, a0 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <8 x i32> poison, i32 %b, i32 0 + %splat = shufflevector <8 x i32> %head, <8 x i32> poison, <8 x i32> zeroinitializer + %x = zext <8 x i16> %a to <8 x i32> + %z = shl <8 x i32> %x, %splat + ret <8 x i32> %z +} + +define <8 x i32> @vwsll_vx_i16_v8i32_sext(<8 x i16> %a, i16 %b) { +; CHECK-LABEL: vwsll_vx_i16_v8i32_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i16_v8i32_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <8 x i16> poison, i16 %b, i32 0 + %splat = shufflevector <8 x i16> %head, <8 x i16> poison, <8 x i32> zeroinitializer + %x = zext <8 x i16> %a to <8 x i32> + %y = sext <8 x i16> %splat to <8 x i32> + %z = shl <8 x i32> %x, %y + ret <8 x i32> %z +} + +define <8 x i32> @vwsll_vx_i16_v8i32_zext(<8 x i16> %a, i16 %b) { +; CHECK-LABEL: vwsll_vx_i16_v8i32_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i16_v8i32_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <8 x i16> poison, i16 %b, i32 0 + %splat = shufflevector <8 x i16> %head, <8 x i16> poison, <8 x i32> zeroinitializer + %x = zext <8 x i16> %a to <8 x i32> + %y = zext <8 x i16> %splat to <8 x i32> + %z = shl <8 x i32> %x, %y + ret <8 x i32> %z +} + +define <8 x i32> @vwsll_vx_i8_v8i32_sext(<8 x i16> %a, i8 %b) { +; CHECK-LABEL: vwsll_vx_i8_v8i32_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf4 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i8_v8i32_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf4 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <8 x i8> poison, i8 %b, i32 0 + %splat = shufflevector <8 x i8> %head, <8 x i8> poison, <8 x i32> zeroinitializer + %x = zext <8 x i16> %a to <8 x i32> + %y = sext <8 x i8> %splat to <8 x i32> + %z = shl <8 x i32> %x, %y + ret <8 x i32> %z +} + +define <8 x i32> @vwsll_vx_i8_v8i32_zext(<8 x i16> %a, i8 %b) { +; CHECK-LABEL: vwsll_vx_i8_v8i32_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf4 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i8_v8i32_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf4 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <8 x i8> poison, i8 %b, i32 0 + %splat = shufflevector <8 x i8> %head, <8 x i8> poison, <8 x i32> zeroinitializer + %x = zext <8 x i16> %a to <8 x i32> + %y = zext <8 x i8> %splat to <8 x i32> + %z = shl <8 x i32> %x, %y + ret <8 x i32> %z +} + +define <8 x i32> @vwsll_vi_v8i32(<8 x i16> %a) { +; CHECK-LABEL: vwsll_vi_v8i32: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsll.vi v8, v10, 2 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vi_v8i32: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsll.vi v8, v10, 2 +; CHECK-ZVBB-NEXT: ret + %x = zext <8 x i16> %a to <8 x i32> + %z = shl <8 x i32> %x, splat (i32 2) + ret <8 x i32> %z +} + +; ============================================================================== +; i8 -> i16 +; ============================================================================== + +define <16 x i16> @vwsll_vv_v16i16_sext(<16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: vwsll_vv_v16i16_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vv_v16i16_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %x = zext <16 x i8> %a to <16 x i16> + %y = sext <16 x i8> %b to <16 x i16> + %z = shl <16 x i16> %x, %y + ret <16 x i16> %z +} + +define <16 x i16> @vwsll_vv_v16i16_zext(<16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: vwsll_vv_v16i16_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vv_v16i16_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %x = zext <16 x i8> %a to <16 x i16> + %y = zext <16 x i8> %b to <16 x i16> + %z = shl <16 x i16> %x, %y + ret <16 x i16> %z +} + +define <16 x i16> @vwsll_vx_i64_v16i16(<16 x i8> %a, i64 %b) { + %head = insertelement <8 x i64> poison, i64 %b, i32 0 + %splat = shufflevector <8 x i64> %head, <8 x i64> poison, <16 x i32> zeroinitializer + %x = zext <16 x i8> %a to <16 x i16> + %y = trunc <16 x i64> %splat to <16 x i16> + %z = shl <16 x i16> %x, %y + ret <16 x i16> %z +} + +define <16 x i16> @vwsll_vx_i32_v16i16(<16 x i8> %a, i32 %b) { +; CHECK-LABEL: vwsll_vx_i32_v16i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vnsrl.wi v8, v12, 0 +; CHECK-NEXT: vsll.vv v8, v10, v8 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i32_v16i16: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v12, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vnsrl.wi v8, v12, 0 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v8 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <16 x i32> poison, i32 %b, i32 0 + %splat = shufflevector <16 x i32> %head, <16 x i32> poison, <16 x i32> zeroinitializer + %x = zext <16 x i8> %a to <16 x i16> + %y = trunc <16 x i32> %splat to <16 x i16> + %z = shl <16 x i16> %x, %y + ret <16 x i16> %z +} + +define <16 x i16> @vwsll_vx_i16_v16i16(<16 x i8> %a, i16 %b) { +; CHECK-LABEL: vwsll_vx_i16_v16i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsll.vx v8, v10, a0 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i16_v16i16: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsll.vx v8, v10, a0 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <16 x i16> poison, i16 %b, i32 0 + %splat = shufflevector <16 x i16> %head, <16 x i16> poison, <16 x i32> zeroinitializer + %x = zext <16 x i8> %a to <16 x i16> + %z = shl <16 x i16> %x, %splat + ret <16 x i16> %z +} + +define <16 x i16> @vwsll_vx_i8_v16i16_sext(<16 x i8> %a, i8 %b) { +; CHECK-LABEL: vwsll_vx_i8_v16i16_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i8_v16i16_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <16 x i8> poison, i8 %b, i32 0 + %splat = shufflevector <16 x i8> %head, <16 x i8> poison, <16 x i32> zeroinitializer + %x = zext <16 x i8> %a to <16 x i16> + %y = sext <16 x i8> %splat to <16 x i16> + %z = shl <16 x i16> %x, %y + ret <16 x i16> %z +} + +define <16 x i16> @vwsll_vx_i8_v16i16_zext(<16 x i8> %a, i8 %b) { +; CHECK-LABEL: vwsll_vx_i8_v16i16_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i8_v16i16_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <16 x i8> poison, i8 %b, i32 0 + %splat = shufflevector <16 x i8> %head, <16 x i8> poison, <16 x i32> zeroinitializer + %x = zext <16 x i8> %a to <16 x i16> + %y = zext <16 x i8> %splat to <16 x i16> + %z = shl <16 x i16> %x, %y + ret <16 x i16> %z +} + +define <16 x i16> @vwsll_vi_v16i16(<16 x i8> %a) { +; CHECK-LABEL: vwsll_vi_v16i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vsll.vi v8, v10, 2 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vi_v16i16: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf2 v10, v8 +; CHECK-ZVBB-NEXT: vsll.vi v8, v10, 2 +; CHECK-ZVBB-NEXT: ret + %x = zext <16 x i8> %a to <16 x i16> + %z = shl <16 x i16> %x, splat (i16 2) + ret <16 x i16> %z +} + +; ============================================================================== +; i8 -> i64 +; ============================================================================== + +define <4 x i64> @vwsll_vv_v4i64_v4i8_sext(<4 x i8> %a, <4 x i8> %b) { +; CHECK-LABEL: vwsll_vv_v4i64_v4i8_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vv_v4i64_v4i8_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf8 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf8 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %x = zext <4 x i8> %a to <4 x i64> + %y = sext <4 x i8> %b to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vv_v4i64_v4i8_zext(<4 x i8> %a, <4 x i8> %b) { +; CHECK-LABEL: vwsll_vv_v4i64_v4i8_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vv_v4i64_v4i8_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf8 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf8 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %x = zext <4 x i8> %a to <4 x i64> + %y = zext <4 x i8> %b to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i64_v4i64_v4i8(<4 x i8> %a, i64 %b) { +; CHECK-LABEL: vwsll_vx_i64_v4i64_v4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vsll.vx v8, v10, a0 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i64_v4i64_v4i8: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf8 v10, v8 +; CHECK-ZVBB-NEXT: vsll.vx v8, v10, a0 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i64> poison, i64 %b, i32 0 + %splat = shufflevector <4 x i64> %head, <4 x i64> poison, <4 x i32> zeroinitializer + %x = zext <4 x i8> %a to <4 x i64> + %z = shl <4 x i64> %x, %splat + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i32_v4i64_v4i8_sext(<4 x i8> %a, i32 %b) { +; CHECK-LABEL: vwsll_vx_i32_v4i64_v4i8_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i32_v4i64_v4i8_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf8 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i32> poison, i32 %b, i32 0 + %splat = shufflevector <4 x i32> %head, <4 x i32> poison, <4 x i32> zeroinitializer + %x = zext <4 x i8> %a to <4 x i64> + %y = sext <4 x i32> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i32_v4i64_v4i8_zext(<4 x i8> %a, i32 %b) { +; CHECK-LABEL: vwsll_vx_i32_v4i64_v4i8_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf2 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i32_v4i64_v4i8_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf8 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf2 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i32> poison, i32 %b, i32 0 + %splat = shufflevector <4 x i32> %head, <4 x i32> poison, <4 x i32> zeroinitializer + %x = zext <4 x i8> %a to <4 x i64> + %y = zext <4 x i32> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i16_v4i64_v4i8_sext(<4 x i8> %a, i16 %b) { +; CHECK-LABEL: vwsll_vx_i16_v4i64_v4i8_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf4 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i16_v4i64_v4i8_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf8 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf4 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i16> poison, i16 %b, i32 0 + %splat = shufflevector <4 x i16> %head, <4 x i16> poison, <4 x i32> zeroinitializer + %x = zext <4 x i8> %a to <4 x i64> + %y = sext <4 x i16> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i16_v4i64_v4i8_zext(<4 x i8> %a, i16 %b) { +; CHECK-LABEL: vwsll_vx_i16_v4i64_v4i8_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf4 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i16_v4i64_v4i8_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf8 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf4 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i16> poison, i16 %b, i32 0 + %splat = shufflevector <4 x i16> %head, <4 x i16> poison, <4 x i32> zeroinitializer + %x = zext <4 x i8> %a to <4 x i64> + %y = zext <4 x i16> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i8_v4i64_v4i8_sext(<4 x i8> %a, i8 %b) { +; CHECK-LABEL: vwsll_vx_i8_v4i64_v4i8_sext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i8_v4i64_v4i8_sext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf8 v10, v8 +; CHECK-ZVBB-NEXT: vsext.vf8 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i8> poison, i8 %b, i32 0 + %splat = shufflevector <4 x i8> %head, <4 x i8> poison, <4 x i32> zeroinitializer + %x = zext <4 x i8> %a to <4 x i64> + %y = sext <4 x i8> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vx_i8_v4i64_v4i8_zext(<4 x i8> %a, i8 %b) { +; CHECK-LABEL: vwsll_vx_i8_v4i64_v4i8_zext: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v12, v9 +; CHECK-NEXT: vsll.vv v8, v10, v12 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vx_i8_v4i64_v4i8_zext: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-ZVBB-NEXT: vmv.v.x v9, a0 +; CHECK-ZVBB-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf8 v10, v8 +; CHECK-ZVBB-NEXT: vzext.vf8 v12, v9 +; CHECK-ZVBB-NEXT: vsll.vv v8, v10, v12 +; CHECK-ZVBB-NEXT: ret + %head = insertelement <4 x i8> poison, i8 %b, i32 0 + %splat = shufflevector <4 x i8> %head, <4 x i8> poison, <4 x i32> zeroinitializer + %x = zext <4 x i8> %a to <4 x i64> + %y = zext <4 x i8> %splat to <4 x i64> + %z = shl <4 x i64> %x, %y + ret <4 x i64> %z +} + +define <4 x i64> @vwsll_vi_v4i64_v4i8(<4 x i8> %a) { +; CHECK-LABEL: vwsll_vi_v4i64_v4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vsll.vi v8, v10, 2 +; CHECK-NEXT: ret +; +; CHECK-ZVBB-LABEL: vwsll_vi_v4i64_v4i8: +; CHECK-ZVBB: # %bb.0: +; CHECK-ZVBB-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-ZVBB-NEXT: vzext.vf8 v10, v8 +; CHECK-ZVBB-NEXT: vsll.vi v8, v10, 2 +; CHECK-ZVBB-NEXT: ret + %x = zext <4 x i8> %a to <4 x i64> + %z = shl <4 x i64> %x, splat (i64 2) + ret <4 x i64> %z +} -- GitLab From 24d528cf4685668d3ad17116846769bed843e933 Mon Sep 17 00:00:00 2001 From: Prabhuk Date: Mon, 1 Apr 2024 23:21:45 -0700 Subject: [PATCH 0164/1690] [MIPS][CallSiteInfo][NFC] Fill CallSiteInfo only when needed (#86847) Argument-register pairs in CallSiteInfo is only needed when EmitCallSiteInfo is on. Currently, the pairs are always pushed to the vector but only used when EmitCallSiteInfo is on. Don't fill the CallSiteInfo vector unless used. Differential Revision: https://reviews.llvm.org/D107108?id=362887 Co-authored-by: Necip Fazil Yildiran --- llvm/lib/Target/Mips/MipsISelLowering.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp index 0a0d40751fcf..1c9c99c6fa94 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.cpp +++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp @@ -3381,7 +3381,7 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // Collect CSInfo about which register passes which parameter. const TargetOptions &Options = DAG.getTarget().Options; - if (Options.SupportsDebugEntryValues) + if (Options.EmitCallSiteInfo) CSInfo.emplace_back(VA.getLocReg(), i); continue; -- GitLab From 93c387df908923f17875ab9cf0463d5f181318bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20K=C3=A9ri?= Date: Tue, 2 Apr 2024 08:55:20 +0200 Subject: [PATCH 0165/1690] [clang][analyzer] Change modeling of `fseek` in StreamChecker. (#86919) Until now function `fseek` returned nonzero on error, this is changed to -1 only. And it does not produce EOF error any more. This complies better with the POSIX standard. --- .../StaticAnalyzer/Checkers/StreamChecker.cpp | 21 +++------ clang/test/Analysis/stream-error.c | 43 ++++++++----------- clang/test/Analysis/stream-note.c | 31 ++++++++++++- 3 files changed, 55 insertions(+), 40 deletions(-) diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp index 902c42a2799b..069e3a633c12 100644 --- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp @@ -1264,15 +1264,10 @@ void StreamChecker::evalFseek(const FnDescription *Desc, const CallEvent &Call, if (!E.Init(Desc, Call, C, State)) return; - const llvm::APSInt *PosV = - C.getSValBuilder().getKnownValue(State, Call.getArgSVal(1)); - const llvm::APSInt *WhenceV = - C.getSValBuilder().getKnownValue(State, Call.getArgSVal(2)); - // Bifurcate the state into failed and non-failed. - // Return zero on success, nonzero on error. - ProgramStateRef StateNotFailed, StateFailed; - std::tie(StateFailed, StateNotFailed) = E.makeRetValAndAssumeDual(State, C); + // Return zero on success, -1 on error. + ProgramStateRef StateNotFailed = E.bindReturnValue(State, C, 0); + ProgramStateRef StateFailed = E.bindReturnValue(State, C, -1); // No failure: Reset the state to opened with no error. StateNotFailed = @@ -1282,12 +1277,10 @@ void StreamChecker::evalFseek(const FnDescription *Desc, const CallEvent &Call, // At error it is possible that fseek fails but sets none of the error flags. // If fseek failed, assume that the file position becomes indeterminate in any // case. - StreamErrorState NewErrS = ErrorNone | ErrorFError; - // Setting the position to start of file never produces EOF error. - if (!(PosV && *PosV == 0 && WhenceV && *WhenceV == SeekSetVal)) - NewErrS = NewErrS | ErrorFEof; - StateFailed = E.setStreamState(StateFailed, - StreamState::getOpened(Desc, NewErrS, true)); + // It is allowed to set the position beyond the end of the file. EOF error + // should not occur. + StateFailed = E.setStreamState( + StateFailed, StreamState::getOpened(Desc, ErrorNone | ErrorFError, true)); C.addTransition(StateFailed, E.getFailureNoteTag(this, C)); } diff --git a/clang/test/Analysis/stream-error.c b/clang/test/Analysis/stream-error.c index 88f7de4234ff..7f9116ff4014 100644 --- a/clang/test/Analysis/stream-error.c +++ b/clang/test/Analysis/stream-error.c @@ -365,27 +365,22 @@ void error_fseek(void) { return; int rc = fseek(F, 1, SEEK_SET); if (rc) { + clang_analyzer_eval(rc == -1); // expected-warning {{TRUE}} int IsFEof = feof(F), IsFError = ferror(F); - // Get feof or ferror or no error. - clang_analyzer_eval(IsFEof || IsFError); - // expected-warning@-1 {{FALSE}} - // expected-warning@-2 {{TRUE}} - clang_analyzer_eval(IsFEof && IsFError); // expected-warning {{FALSE}} + // Get ferror or no error. + clang_analyzer_eval(IsFError); // expected-warning {{FALSE}} \ + // expected-warning {{TRUE}} + clang_analyzer_eval(IsFEof); // expected-warning {{FALSE}} // Error flags should not change. - if (IsFEof) - clang_analyzer_eval(feof(F)); // expected-warning {{TRUE}} - else - clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} + clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} if (IsFError) - clang_analyzer_eval(ferror(F)); // expected-warning {{TRUE}} - else - clang_analyzer_eval(ferror(F)); // expected-warning {{FALSE}} + clang_analyzer_eval(ferror(F)); // expected-warning {{TRUE}} } else { - clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} - clang_analyzer_eval(ferror(F)); // expected-warning {{FALSE}} + clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} + clang_analyzer_eval(ferror(F)); // expected-warning {{FALSE}} // Error flags should not change. - clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} - clang_analyzer_eval(ferror(F)); // expected-warning {{FALSE}} + clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} + clang_analyzer_eval(ferror(F)); // expected-warning {{FALSE}} } fclose(F); } @@ -396,15 +391,13 @@ void error_fseeko(void) { return; int rc = fseeko(F, 1, SEEK_SET); if (rc) { - int IsFEof = feof(F), IsFError = ferror(F); - // Get feof or ferror or no error. - clang_analyzer_eval(IsFEof || IsFError); - // expected-warning@-1 {{FALSE}} - // expected-warning@-2 {{TRUE}} - clang_analyzer_eval(IsFEof && IsFError); // expected-warning {{FALSE}} + // Get ferror or no error. + clang_analyzer_eval(ferror(F)); // expected-warning {{FALSE}} \ + // expected-warning {{TRUE}} + clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} } else { - clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} - clang_analyzer_eval(ferror(F)); // expected-warning {{FALSE}} + clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} + clang_analyzer_eval(ferror(F)); // expected-warning {{FALSE}} } fclose(F); } @@ -414,7 +407,7 @@ void error_fseek_0(void) { if (!F) return; int rc = fseek(F, 0, SEEK_SET); - if (rc) { + if (rc == -1) { int IsFEof = feof(F), IsFError = ferror(F); // Get ferror or no error, but not feof. clang_analyzer_eval(IsFError); diff --git a/clang/test/Analysis/stream-note.c b/clang/test/Analysis/stream-note.c index f77cd4aa6284..54ea699f4667 100644 --- a/clang/test/Analysis/stream-note.c +++ b/clang/test/Analysis/stream-note.c @@ -226,10 +226,39 @@ void check_indeterminate_fseek(void) { return; int Ret = fseek(F, 1, SEEK_SET); // expected-note {{Assuming this stream operation fails}} if (Ret) { // expected-note {{Taking true branch}} \ - // expected-note {{'Ret' is not equal to 0}} + // expected-note {{'Ret' is -1}} char Buf[2]; fwrite(Buf, 1, 2, F); // expected-warning {{might be 'indeterminate'}} \ // expected-note {{might be 'indeterminate'}} } fclose(F); } + +void error_fseek_ftell(void) { + FILE *F = fopen("file", "r"); + if (!F) // expected-note {{Taking false branch}} \ + // expected-note {{'F' is non-null}} + return; + fseek(F, 0, SEEK_END); // expected-note {{Assuming this stream operation fails}} + long size = ftell(F); // expected-warning {{might be 'indeterminate'}} \ + // expected-note {{might be 'indeterminate'}} + if (size == -1) { + fclose(F); + return; + } + if (size == 1) + fprintf(F, "abcd"); + fclose(F); +} + +void error_fseek_read_eof(void) { + FILE *F = fopen("file", "r"); + if (!F) + return; + if (fseek(F, 22, SEEK_SET) == -1) { + fclose(F); + return; + } + fgetc(F); // no warning + fclose(F); +} -- GitLab From 8bb9443333e0117ab61feecce9de339b11b924fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= Date: Tue, 2 Apr 2024 09:01:24 +0200 Subject: [PATCH 0166/1690] [GlobalIsel] Combine G_EXTRACT_VECTOR_ELT (#85321) preliminary steps --- .../llvm/CodeGen/GlobalISel/CombinerHelper.h | 25 ++ .../CodeGen/GlobalISel/GenericMachineInstrs.h | 41 +++ .../include/llvm/Target/GlobalISel/Combine.td | 230 +++++++++++- llvm/lib/CodeGen/GlobalISel/CMakeLists.txt | 1 + .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 8 + .../GlobalISel/CombinerHelperVectorOps.cpp | 326 ++++++++++++++++++ .../GlobalISel/combine-extract-vec-elt.mir | 299 +++++++++++++++- .../CodeGen/AArch64/extract-vector-elt.ll | 18 +- .../AArch64/extractvector-oob-load.mir | 7 +- 9 files changed, 930 insertions(+), 25 deletions(-) create mode 100644 llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 28d9cf6260d6..3af32043391f 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -594,6 +594,10 @@ public: /// This variant does not erase \p MI after calling the build function. void applyBuildFnNoErase(MachineInstr &MI, BuildFnTy &MatchInfo); + /// Use a function which takes in a MachineIRBuilder to perform a combine. + /// By default, it erases the instruction \p MI from the function. + void applyBuildFnMO(const MachineOperand &MO, BuildFnTy &MatchInfo); + bool matchOrShiftToFunnelShift(MachineInstr &MI, BuildFnTy &MatchInfo); bool matchFunnelShiftToRotate(MachineInstr &MI); void applyFunnelShiftToRotate(MachineInstr &MI); @@ -823,6 +827,27 @@ public: /// Combine addos. bool matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo); + /// Combine extract vector element. + bool matchExtractVectorElement(MachineInstr &MI, BuildFnTy &MatchInfo); + + /// Combine extract vector element with freeze on the vector register. + bool matchExtractVectorElementWithFreeze(const MachineOperand &MO, + BuildFnTy &MatchInfo); + + /// Combine extract vector element with a build vector on the vector register. + bool matchExtractVectorElementWithBuildVector(const MachineOperand &MO, + BuildFnTy &MatchInfo); + + /// Combine extract vector element with a build vector trunc on the vector + /// register. + bool matchExtractVectorElementWithBuildVectorTrunc(const MachineOperand &MO, + BuildFnTy &MatchInfo); + + /// Combine extract vector element with a insert vector element on the vector + /// register and different indices. + bool matchExtractVectorElementWithDifferentIndices(const MachineOperand &MO, + BuildFnTy &MatchInfo); + private: /// Checks for legality of an indexed variant of \p LdSt. bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h index 261cfcf504d5..25e47114e4a3 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h @@ -286,6 +286,14 @@ public: } }; +/// Represents a G_BUILD_VECTOR_TRUNC. +class GBuildVectorTrunc : public GMergeLikeInstr { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC; + } +}; + /// Represents a G_PTR_ADD. class GPtrAdd : public GenericMachineInstr { public: @@ -739,6 +747,39 @@ public: }; }; +/// Represents an extract vector element. +class GExtractVectorElement : public GenericMachineInstr { +public: + Register getVectorReg() const { return getOperand(1).getReg(); } + Register getIndexReg() const { return getOperand(2).getReg(); } + + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT; + } +}; + +/// Represents an insert vector element. +class GInsertVectorElement : public GenericMachineInstr { +public: + Register getVectorReg() const { return getOperand(1).getReg(); } + Register getElementReg() const { return getOperand(2).getReg(); } + Register getIndexReg() const { return getOperand(3).getReg(); } + + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT; + } +}; + +/// Represents a freeze. +class GFreeze : public GenericMachineInstr { +public: + Register getSourceReg() const { return getOperand(1).getReg(); } + + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_FREEZE; + } +}; + } // namespace llvm #endif // LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index 72d3c0ea69bc..778ff7e437eb 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -1305,6 +1305,200 @@ def match_addos : GICombineRule< [{ return Helper.matchAddOverflow(*${root}, ${matchinfo}); }]), (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>; +def match_extract_of_element_undef_vector: GICombineRule < + (defs root:$root), + (match (G_IMPLICIT_DEF $vector), + (G_EXTRACT_VECTOR_ELT $root, $vector, $idx)), + (apply (G_IMPLICIT_DEF $root)) +>; + +def match_extract_of_element_undef_index: GICombineRule < + (defs root:$root), + (match (G_IMPLICIT_DEF $idx), + (G_EXTRACT_VECTOR_ELT $root, $vector, $idx)), + (apply (G_IMPLICIT_DEF $root)) +>; + +def match_extract_of_element : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root, + [{ return Helper.matchExtractVectorElement(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>; + +def extract_vector_element_not_const : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_INSERT_VECTOR_ELT $src, $x, $value, $idx), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx)), + (apply (GIReplaceReg $root, $value))>; + +def extract_vector_element_different_indices : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_INSERT_VECTOR_ELT $src, $x, $value, $idx2), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx1), + [{ return Helper.matchExtractVectorElementWithDifferentIndices(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector2 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector3 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector4 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector5 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector6 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector7 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector8 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector9 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector10 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector11 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector12 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector13 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector14 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector15 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector16 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_trunc2 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR_TRUNC $src, $x, $y), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_trunc3 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_trunc4 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_trunc5 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_trunc6 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b, $c), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_trunc7 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b, $c, $d), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_trunc8 : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b, $c, $d, $e), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def extract_vector_element_freeze : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_FREEZE $src, $input), + (G_EXTRACT_VECTOR_ELT $root, $src, $idx), + [{ return Helper.matchExtractVectorElementWithFreeze(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + // Combines concat operations def concat_matchinfo : GIDefMatchData<"SmallVector">; def combine_concat_vector : GICombineRule< @@ -1313,6 +1507,37 @@ def combine_concat_vector : GICombineRule< [{ return Helper.matchCombineConcatVectors(*${root}, ${matchinfo}); }]), (apply [{ Helper.applyCombineConcatVectors(*${root}, ${matchinfo}); }])>; +// match_extract_of_element must be the first! +def vector_ops_combines: GICombineGroup<[ +match_extract_of_element_undef_vector, +match_extract_of_element_undef_index, +match_extract_of_element, +extract_vector_element_not_const, +extract_vector_element_different_indices, +extract_vector_element_build_vector2, +extract_vector_element_build_vector3, +extract_vector_element_build_vector4, +extract_vector_element_build_vector5, +extract_vector_element_build_vector7, +extract_vector_element_build_vector8, +extract_vector_element_build_vector9, +extract_vector_element_build_vector10, +extract_vector_element_build_vector11, +extract_vector_element_build_vector12, +extract_vector_element_build_vector13, +extract_vector_element_build_vector14, +extract_vector_element_build_vector15, +extract_vector_element_build_vector16, +extract_vector_element_build_vector_trunc2, +extract_vector_element_build_vector_trunc3, +extract_vector_element_build_vector_trunc4, +extract_vector_element_build_vector_trunc5, +extract_vector_element_build_vector_trunc6, +extract_vector_element_build_vector_trunc7, +extract_vector_element_build_vector_trunc8, +extract_vector_element_freeze +]>; + // FIXME: These should use the custom predicate feature once it lands. def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero, undef_to_negative_one, @@ -1368,8 +1593,9 @@ def fma_combines : GICombineGroup<[combine_fadd_fmul_to_fmad_or_fma, def constant_fold_binops : GICombineGroup<[constant_fold_binop, constant_fold_fp_binop]>; -def all_combines : GICombineGroup<[trivial_combines, insert_vec_elt_combines, - extract_vec_elt_combines, combines_for_extload, combine_extracted_vector_load, +def all_combines : GICombineGroup<[trivial_combines, vector_ops_combines, + insert_vec_elt_combines, extract_vec_elt_combines, combines_for_extload, + combine_extracted_vector_load, undef_combines, identity_combines, phi_combines, simplify_add_to_sub, hoist_logic_op_with_same_opcode_hands, shifts_too_big, reassocs, ptr_add_immed_chain, diff --git a/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt b/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt index 46e6c6df5998..54ac7f72011a 100644 --- a/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt +++ b/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt @@ -6,6 +6,7 @@ add_llvm_component_library(LLVMGlobalISel GlobalISel.cpp Combiner.cpp CombinerHelper.cpp + CombinerHelperVectorOps.cpp GIMatchTableExecutor.cpp GISelChangeObserver.cpp IRTranslator.cpp diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index 98e7c73a801f..5cf7a33a5f67 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -4058,6 +4058,14 @@ void CombinerHelper::applyBuildFn( MI.eraseFromParent(); } +void CombinerHelper::applyBuildFnMO(const MachineOperand &MO, + BuildFnTy &MatchInfo) { + MachineInstr *Root = getDefIgnoringCopies(MO.getReg(), MRI); + Builder.setInstrAndDebugLoc(*Root); + MatchInfo(Builder); + Root->eraseFromParent(); +} + void CombinerHelper::applyBuildFnNoErase( MachineInstr &MI, std::function &MatchInfo) { MatchInfo(Builder); diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp new file mode 100644 index 000000000000..123bf21f657c --- /dev/null +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp @@ -0,0 +1,326 @@ +//===- CombinerHelperVectorOps.cpp-----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements CombinerHelper for G_EXTRACT_VECTOR_ELT. +// +//===----------------------------------------------------------------------===// +#include "llvm/CodeGen/GlobalISel/CombinerHelper.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" +#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" +#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" +#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/LowLevelTypeUtils.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/Support/Casting.h" +#include + +#define DEBUG_TYPE "gi-combiner" + +using namespace llvm; +using namespace MIPatternMatch; + +bool CombinerHelper::matchExtractVectorElement(MachineInstr &MI, + BuildFnTy &MatchInfo) { + GExtractVectorElement *Extract = cast(&MI); + + Register Dst = Extract->getReg(0); + Register Vector = Extract->getVectorReg(); + Register Index = Extract->getIndexReg(); + LLT DstTy = MRI.getType(Dst); + LLT VectorTy = MRI.getType(Vector); + + // The vector register can be def'd by various ops that have vector as its + // type. They can all be used for constant folding, scalarizing, + // canonicalization, or combining based on symmetry. + // + // vector like ops + // * build vector + // * build vector trunc + // * shuffle vector + // * splat vector + // * concat vectors + // * insert/extract vector element + // * insert/extract subvector + // * vector loads + // * scalable vector loads + // + // compute like ops + // * binary ops + // * unary ops + // * exts and truncs + // * casts + // * fneg + // * select + // * phis + // * cmps + // * freeze + // * bitcast + // * undef + + // We try to get the value of the Index register. + std::optional MaybeIndex = + getIConstantVRegValWithLookThrough(Index, MRI); + std::optional IndexC = std::nullopt; + + if (MaybeIndex) + IndexC = MaybeIndex->Value; + + // Fold extractVectorElement(Vector, TOOLARGE) -> undef + if (IndexC && VectorTy.isFixedVector() && + IndexC->getZExtValue() >= VectorTy.getNumElements() && + isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) { + // For fixed-length vectors, it's invalid to extract out-of-range elements. + MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Dst); }; + return true; + } + + return false; +} + +bool CombinerHelper::matchExtractVectorElementWithDifferentIndices( + const MachineOperand &MO, BuildFnTy &MatchInfo) { + MachineInstr *Root = getDefIgnoringCopies(MO.getReg(), MRI); + GExtractVectorElement *Extract = cast(Root); + + // + // %idx1:_(s64) = G_CONSTANT i64 1 + // %idx2:_(s64) = G_CONSTANT i64 2 + // %insert:_(<2 x s32>) = G_INSERT_VECTOR_ELT_ELT %bv(<2 x s32>), + // %value(s32), %idx2(s64) %extract:_(s32) = G_EXTRACT_VECTOR_ELT %insert(<2 + // x s32>), %idx1(s64) + // + // --> + // + // %insert:_(<2 x s32>) = G_INSERT_VECTOR_ELT_ELT %bv(<2 x s32>), + // %value(s32), %idx2(s64) %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x + // s32>), %idx1(s64) + // + // + + Register Index = Extract->getIndexReg(); + + // We try to get the value of the Index register. + std::optional MaybeIndex = + getIConstantVRegValWithLookThrough(Index, MRI); + std::optional IndexC = std::nullopt; + + if (!MaybeIndex) + return false; + else + IndexC = MaybeIndex->Value; + + Register Vector = Extract->getVectorReg(); + + GInsertVectorElement *Insert = + getOpcodeDef(Vector, MRI); + if (!Insert) + return false; + + Register Dst = Extract->getReg(0); + + std::optional MaybeInsertIndex = + getIConstantVRegValWithLookThrough(Insert->getIndexReg(), MRI); + + if (MaybeInsertIndex && MaybeInsertIndex->Value != *IndexC) { + // There is no one-use check. We have to keep the insert. When both Index + // registers are constants and not equal, we can look into the Vector + // register of the insert. + MatchInfo = [=](MachineIRBuilder &B) { + B.buildExtractVectorElement(Dst, Insert->getVectorReg(), Index); + }; + return true; + } + + return false; +} + +bool CombinerHelper::matchExtractVectorElementWithFreeze( + const MachineOperand &MO, BuildFnTy &MatchInfo) { + MachineInstr *Root = getDefIgnoringCopies(MO.getReg(), MRI); + GExtractVectorElement *Extract = cast(Root); + + Register Vector = Extract->getVectorReg(); + + // + // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) + // %freeze:_(<2 x s32>) = G_FREEZE %bv(<2 x s32>) + // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) + // + // --> + // + // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) + // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) + // %freeze:_(s32) = G_FREEZE %extract(s32) + // + // + + // For G_FREEZE, the input and the output types are identical. Moving the + // freeze from the Vector into the front of the extract preserves the freeze + // semantics. The result is still freeze'd. Furthermore, the Vector register + // becomes easier to analyze. A build vector could have been hidden behind the + // freeze. + + // We expect a freeze on the Vector register. + GFreeze *Freeze = getOpcodeDef(Vector, MRI); + if (!Freeze) + return false; + + Register Dst = Extract->getReg(0); + LLT DstTy = MRI.getType(Dst); + + // We first have to check for one-use and legality of the freeze. + // The type of the extractVectorElement did not change. + if (!MRI.hasOneNonDBGUse(Freeze->getReg(0)) || + !isLegalOrBeforeLegalizer({TargetOpcode::G_FREEZE, {DstTy}})) + return false; + + Register Index = Extract->getIndexReg(); + + // We move the freeze from the Vector register in front of the + // extractVectorElement. + MatchInfo = [=](MachineIRBuilder &B) { + auto Extract = + B.buildExtractVectorElement(DstTy, Freeze->getSourceReg(), Index); + B.buildFreeze(Dst, Extract); + }; + + return true; +} + +bool CombinerHelper::matchExtractVectorElementWithBuildVector( + const MachineOperand &MO, BuildFnTy &MatchInfo) { + MachineInstr *Root = getDefIgnoringCopies(MO.getReg(), MRI); + GExtractVectorElement *Extract = cast(Root); + + // + // %zero:_(s64) = G_CONSTANT i64 0 + // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) + // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %zero(s64) + // + // --> + // + // %extract:_(32) = COPY %arg1(s32) + // + // + // + // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) + // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) + // + // --> + // + // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) + // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) + // + + Register Vector = Extract->getVectorReg(); + + // We expect a buildVector on the Vector register. + GBuildVector *Build = getOpcodeDef(Vector, MRI); + if (!Build) + return false; + + LLT VectorTy = MRI.getType(Vector); + + // There is a one-use check. There are more combines on build vectors. + EVT Ty(getMVTForLLT(VectorTy)); + if (!MRI.hasOneNonDBGUse(Build->getReg(0)) || + !getTargetLowering().aggressivelyPreferBuildVectorSources(Ty)) + return false; + + Register Index = Extract->getIndexReg(); + + // If the Index is constant, then we can extract the element from the given + // offset. + std::optional MaybeIndex = + getIConstantVRegValWithLookThrough(Index, MRI); + if (!MaybeIndex) + return false; + + // We now know that there is a buildVector def'd on the Vector register and + // the index is const. The combine will succeed. + + Register Dst = Extract->getReg(0); + + MatchInfo = [=](MachineIRBuilder &B) { + B.buildCopy(Dst, Build->getSourceReg(MaybeIndex->Value.getZExtValue())); + }; + + return true; +} + +bool CombinerHelper::matchExtractVectorElementWithBuildVectorTrunc( + const MachineOperand &MO, BuildFnTy &MatchInfo) { + MachineInstr *Root = getDefIgnoringCopies(MO.getReg(), MRI); + GExtractVectorElement *Extract = cast(Root); + + // + // %zero:_(s64) = G_CONSTANT i64 0 + // %bv:_(<2 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64) + // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %zero(s64) + // + // --> + // + // %extract:_(32) = G_TRUNC %arg1(s64) + // + // + // + // %bv:_(<2 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64) + // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) + // + // --> + // + // %bv:_(<2 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64) + // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) + // + + Register Vector = Extract->getVectorReg(); + + // We expect a buildVectorTrunc on the Vector register. + GBuildVectorTrunc *Build = getOpcodeDef(Vector, MRI); + if (!Build) + return false; + + LLT VectorTy = MRI.getType(Vector); + + // There is a one-use check. There are more combines on build vectors. + EVT Ty(getMVTForLLT(VectorTy)); + if (!MRI.hasOneNonDBGUse(Build->getReg(0)) || + !getTargetLowering().aggressivelyPreferBuildVectorSources(Ty)) + return false; + + Register Index = Extract->getIndexReg(); + + // If the Index is constant, then we can extract the element from the given + // offset. + std::optional MaybeIndex = + getIConstantVRegValWithLookThrough(Index, MRI); + if (!MaybeIndex) + return false; + + // We now know that there is a buildVectorTrunc def'd on the Vector register + // and the index is const. The combine will succeed. + + Register Dst = Extract->getReg(0); + LLT DstTy = MRI.getType(Dst); + LLT SrcTy = MRI.getType(Build->getSourceReg(0)); + + // For buildVectorTrunc, the inputs are truncated. + if (!isLegalOrBeforeLegalizer({TargetOpcode::G_TRUNC, {DstTy, SrcTy}})) + return false; + + MatchInfo = [=](MachineIRBuilder &B) { + B.buildTrunc(Dst, Build->getSourceReg(MaybeIndex->Value.getZExtValue())); + }; + + return true; +} diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir index a2116ccc7671..c2a38e26676c 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir @@ -192,8 +192,8 @@ body: | ... --- +# This test checks that this combine runs after the insertvec->build_vector name: extract_from_insert -alignment: 4 tracksRegLiveness: true liveins: - { reg: '$x0' } @@ -203,8 +203,6 @@ frameInfo: body: | bb.1: liveins: $x0, $x1 - ; This test checks that this combine runs after the insertvec->build_vector - ; combine. ; CHECK-LABEL: name: extract_from_insert ; CHECK: liveins: $x0, $x1 ; CHECK-NEXT: {{ $}} @@ -247,3 +245,298 @@ body: | RET_ReallyLR implicit $x0 ... +--- +name: extract_from_vector_undef +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: extract_from_vector_undef + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %extract:_(s64) = G_IMPLICIT_DEF + ; CHECK-NEXT: $x0 = COPY %extract(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %vec:_(<2 x s64>) = G_IMPLICIT_DEF + %idx:_(s32) = G_CONSTANT i32 -2 + %extract:_(s64) = G_EXTRACT_VECTOR_ELT %vec(<2 x s64>), %idx(s32) + $x0 = COPY %extract(s64) + RET_ReallyLR implicit $x0 + +... +--- +name: extract_from_index_undef +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + ; CHECK-LABEL: name: extract_from_index_undef + ; CHECK: %extract:_(s64) = G_IMPLICIT_DEF + ; CHECK-NEXT: $x0 = COPY %extract(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %vec:_(<2 x s64>) = COPY $q0 + %idx:_(s32) = G_IMPLICIT_DEF + %extract:_(s64) = G_EXTRACT_VECTOR_ELT %vec(<2 x s64>), %idx(s32) + $x0 = COPY %extract(s64) + RET_ReallyLR implicit $x0 + +... +--- +name: extract_from_index_too_large +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: extract_from_index_too_large + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %extract:_(s64) = G_IMPLICIT_DEF + ; CHECK-NEXT: $x0 = COPY %extract(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %vec:_(<2 x s64>) = COPY $q0 + %idx:_(s32) = G_CONSTANT i32 3000 + %extract:_(s64) = G_EXTRACT_VECTOR_ELT %vec(<2 x s64>), %idx(s32) + $x0 = COPY %extract(s64) + RET_ReallyLR implicit $x0 + +... +--- +name: extract_with_freeze +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: extract_with_freeze + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %vec:_(<2 x s64>) = COPY $q0 + ; CHECK-NEXT: %idx:_(s32) = COPY $w1 + ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT %vec(<2 x s64>), %idx(s32) + ; CHECK-NEXT: %extract:_(s64) = G_FREEZE [[EVEC]] + ; CHECK-NEXT: $x0 = COPY %extract(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %vec:_(<2 x s64>) = COPY $q0 + %idx:_(s32) = COPY $w1 + %fvec:_(<2 x s64>) = G_FREEZE %vec + %extract:_(s64) = G_EXTRACT_VECTOR_ELT %fvec(<2 x s64>), %idx(s32) + $x0 = COPY %extract(s64) + RET_ReallyLR implicit $x0 + +... +--- +name: extract_from_insert_symmetry +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: extract_from_insert_symmetry + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %element:_(s64) = COPY $x1 + ; CHECK-NEXT: $x0 = COPY %element(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %vec:_(<2 x s64>) = COPY $q0 + %idx:_(s32) = COPY $w1 + %element:_(s64) = COPY $x1 + %invec:_(<2 x s64>) = G_INSERT_VECTOR_ELT %vec(<2 x s64>), %element(s64), %idx(s32) + %extract:_(s64) = G_EXTRACT_VECTOR_ELT %invec(<2 x s64>), %idx(s32) + $x0 = COPY %extract(s64) + RET_ReallyLR implicit $x0 + +... +--- +name: extract_from_insert_with_different_consts +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: extract_from_insert_with_different_consts + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %vec:_(<2 x s64>) = COPY $q0 + ; CHECK-NEXT: %idx2:_(s32) = G_CONSTANT i32 1 + ; CHECK-NEXT: %extract:_(s64) = G_EXTRACT_VECTOR_ELT %vec(<2 x s64>), %idx2(s32) + ; CHECK-NEXT: $x0 = COPY %extract(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %vec:_(<2 x s64>) = COPY $q0 + %idx:_(s32) = G_CONSTANT i32 0 + %idx2:_(s32) = G_CONSTANT i32 1 + %element:_(s64) = COPY $x1 + %invec:_(<2 x s64>) = G_INSERT_VECTOR_ELT %vec(<2 x s64>), %element(s64), %idx(s32) + %extract:_(s64) = G_EXTRACT_VECTOR_ELT %invec(<2 x s64>), %idx2(s32) + $x0 = COPY %extract(s64) + RET_ReallyLR implicit $x0 + +... +--- +name: extract_from_build_vector_non_const +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: extract_from_build_vector_non_const + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %idx:_(s32) = COPY $w0 + ; CHECK-NEXT: %arg1:_(s64) = COPY $x0 + ; CHECK-NEXT: %arg2:_(s64) = COPY $x1 + ; CHECK-NEXT: %bv:_(<2 x s64>) = G_BUILD_VECTOR %arg1(s64), %arg2(s64) + ; CHECK-NEXT: %extract:_(s64) = G_EXTRACT_VECTOR_ELT %bv(<2 x s64>), %idx(s32) + ; CHECK-NEXT: $x0 = COPY %extract(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %vec:_(<2 x s64>) = COPY $q0 + %idx:_(s32) = COPY $w0 + %arg1:_(s64) = COPY $x0 + %arg2:_(s64) = COPY $x1 + %bv:_(<2 x s64>) = G_BUILD_VECTOR %arg1(s64), %arg2(s64) + %extract:_(s64) = G_EXTRACT_VECTOR_ELT %bv(<2 x s64>), %idx(s32) + $x0 = COPY %extract(s64) + RET_ReallyLR implicit $x0 + +... +--- +name: extract_from_build_vector_const +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: extract_from_build_vector_const + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %arg1:_(s64) = COPY $x0 + ; CHECK-NEXT: $x0 = COPY %arg1(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %vec:_(<2 x s64>) = COPY $q0 + %idx:_(s32) = G_CONSTANT i32 0 + %arg1:_(s64) = COPY $x0 + %arg2:_(s64) = COPY $x1 + %bv:_(<2 x s64>) = G_BUILD_VECTOR %arg1(s64), %arg2(s64) + %extract:_(s64) = G_EXTRACT_VECTOR_ELT %bv(<2 x s64>), %idx(s32) + $x0 = COPY %extract(s64) + RET_ReallyLR implicit $x0 + +... +--- +name: extract_from_build_vector_trunc_const2 +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: extract_from_build_vector_trunc_const2 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %arg1:_(s64) = COPY $x0 + ; CHECK-NEXT: %extract:_(s32) = G_TRUNC %arg1(s64) + ; CHECK-NEXT: $w0 = COPY %extract(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %vec:_(<2 x s64>) = COPY $q0 + %arg1:_(s64) = COPY $x0 + %arg2:_(s64) = COPY $x1 + %arg3:_(s64) = COPY $x0 + %arg4:_(s64) = COPY $x1 + %idx:_(s32) = G_CONSTANT i32 0 + %bv:_(<4 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64), %arg3(s64), %arg4(s64) + %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<4 x s32>), %idx(s32) + $w0 = COPY %extract(s32) + RET_ReallyLR implicit $x0 +... +--- +name: extract_from_build_vector_trunc2 +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: extract_from_build_vector_trunc2 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %arg1:_(s64) = COPY $x0 + ; CHECK-NEXT: %arg2:_(s64) = COPY $x1 + ; CHECK-NEXT: %idx:_(s32) = COPY $w0 + ; CHECK-NEXT: %bv:_(<2 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64) + ; CHECK-NEXT: %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %idx(s32) + ; CHECK-NEXT: $w0 = COPY %extract(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %arg1:_(s64) = COPY $x0 + %arg2:_(s64) = COPY $x1 + %idx:_(s32) = COPY $w0 + %bv:_(<2 x s32>) = G_BUILD_VECTOR_TRUNC %arg1(s64), %arg2(s64) + %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %idx(s32) + $w0 = COPY %extract(s32) + RET_ReallyLR implicit $x0 +... +--- +name: extract_from_build_vector_trunc_const3 +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: extract_from_build_vector_trunc_const3 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %arg1:_(s128) = COPY $q0 + ; CHECK-NEXT: %extract:_(s64) = G_TRUNC %arg1(s128) + ; CHECK-NEXT: $x0 = COPY %extract(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %arg1:_(s128) = COPY $q0 + %arg2:_(s128) = COPY $q1 + %idx:_(s32) = G_CONSTANT i32 0 + %bv:_(<2 x s64>) = G_BUILD_VECTOR_TRUNC %arg1(s128), %arg2(s128) + %extract:_(s64) = G_EXTRACT_VECTOR_ELT %bv(<2 x s64>), %idx(s32) + $x0 = COPY %extract(s64) + RET_ReallyLR implicit $x0 +... +--- diff --git a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll index c5c525a15ad9..504222e0036e 100644 --- a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll +++ b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll @@ -25,20 +25,9 @@ entry: } define i64 @extract_v2i64_undef_vector(<2 x i64> %a, i32 %c) { -; CHECK-SD-LABEL: extract_v2i64_undef_vector: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: extract_v2i64_undef_vector: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: sub sp, sp, #16 -; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 -; CHECK-GI-NEXT: mov w9, w0 -; CHECK-GI-NEXT: mov x8, sp -; CHECK-GI-NEXT: and x9, x9, #0x1 -; CHECK-GI-NEXT: ldr x0, [x8, x9, lsl #3] -; CHECK-GI-NEXT: add sp, sp, #16 -; CHECK-GI-NEXT: ret +; CHECK-LABEL: extract_v2i64_undef_vector: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ret entry: %d = extractelement <2 x i64> undef, i32 %c ret i64 %d @@ -130,7 +119,6 @@ define i64 @extract_v2i64_extract_of_insert_different_const(<2 x i64> %a, i64 %e ; ; CHECK-GI-LABEL: extract_v2i64_extract_of_insert_different_const: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: mov v0.d[0], x0 ; CHECK-GI-NEXT: mov d0, v0.d[1] ; CHECK-GI-NEXT: fmov x0, d0 ; CHECK-GI-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/extractvector-oob-load.mir b/llvm/test/CodeGen/AArch64/extractvector-oob-load.mir index e8c5819e75e0..e7e8c9399109 100644 --- a/llvm/test/CodeGen/AArch64/extractvector-oob-load.mir +++ b/llvm/test/CodeGen/AArch64/extractvector-oob-load.mir @@ -22,11 +22,8 @@ body: | ; CHECK-LABEL: name: f ; CHECK: liveins: $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16 - ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64) - ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p0) :: (load (s64)) - ; CHECK-NEXT: $x0 = COPY [[LOAD]](s64) + ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF + ; CHECK-NEXT: $x0 = COPY [[DEF]](s64) ; CHECK-NEXT: RET_ReallyLR implicit $x0 %0:_(p0) = COPY $x0 %3:_(s64) = G_CONSTANT i64 224567957 -- GitLab From e47a81c1d2830dda45a561e2c092ebb0c868ed27 Mon Sep 17 00:00:00 2001 From: Sven van Haastregt Date: Tue, 2 Apr 2024 09:31:38 +0200 Subject: [PATCH 0167/1690] [OpenCL] Fix BIenqueue_kernel fallthrough (#83238) Handling of the `BIenqueue_kernel` builtin must not fallthrough to the `BIget_kernel_work_group_size` builtin, as these builtins have no common functionality. --- clang/lib/CodeGen/CGBuiltin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index bb007231c0b7..483f9c268599 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -5835,7 +5835,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, EmitLifetimeEnd(TmpSize, TmpPtr); return Call; } - [[fallthrough]]; + llvm_unreachable("Unexpected enqueue_kernel signature"); } // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block // parameter. -- GitLab From f6c87be1dd24a121d7eccd6b91ca808ecdf80356 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Tue, 2 Apr 2024 00:33:50 -0700 Subject: [PATCH 0168/1690] [Github] Fix typo in PR code formatting job The recent change to split the PR code formatting job accidentally misspelled the repository field when specifying the repository to fetch the code formatting utils from. This patch fixes the spelling so that the job does not throw a warning and clones the tools from the specified repository. --- .github/workflows/pr-code-format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml index 54dfe3aadbb4..10b18f245d89 100644 --- a/.github/workflows/pr-code-format.yml +++ b/.github/workflows/pr-code-format.yml @@ -33,7 +33,7 @@ jobs: - name: Fetch code formatting utils uses: actions/checkout@v4 with: - reository: ${{ github.repository }} + repository: ${{ github.repository }} ref: ${{ github.base_ref }} sparse-checkout: | llvm/utils/git/requirements_formatting.txt -- GitLab From fa8dc363506893eb9371dd3b7590f41fa9a7174a Mon Sep 17 00:00:00 2001 From: elhewaty Date: Tue, 2 Apr 2024 09:49:31 +0200 Subject: [PATCH 0169/1690] [IR] Fix crashes caused by #85592 (#87169) This patch fixes the crash caused by the pull request: https://github.com/llvm/llvm-project/pull/85592 --- llvm/lib/IR/Operator.cpp | 5 +++-- llvm/test/Transforms/FunctionAttrs/noundef.ll | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/llvm/lib/IR/Operator.cpp b/llvm/lib/IR/Operator.cpp index 495769279e33..7b4449cd825f 100644 --- a/llvm/lib/IR/Operator.cpp +++ b/llvm/lib/IR/Operator.cpp @@ -28,8 +28,9 @@ bool Operator::hasPoisonGeneratingFlags() const { return OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap(); } case Instruction::Trunc: { - auto *TI = dyn_cast(this); - return TI->hasNoUnsignedWrap() || TI->hasNoSignedWrap(); + if (auto *TI = dyn_cast(this)) + return TI->hasNoUnsignedWrap() || TI->hasNoSignedWrap(); + return false; } case Instruction::UDiv: case Instruction::SDiv: diff --git a/llvm/test/Transforms/FunctionAttrs/noundef.ll b/llvm/test/Transforms/FunctionAttrs/noundef.ll index 946b562f3955..9ab37082a303 100644 --- a/llvm/test/Transforms/FunctionAttrs/noundef.ll +++ b/llvm/test/Transforms/FunctionAttrs/noundef.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 ; RUN: opt < %s -passes='function-attrs' -S | FileCheck %s +@g_var = external global [0 x i8] + define i32 @test_ret_constant() { ; CHECK-LABEL: define noundef i32 @test_ret_constant( ; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { @@ -152,3 +154,15 @@ define i32 @test_ret_constant_msan() sanitize_memory { ; ret i32 0 } + +define i64 @test_trunc_with_constexpr() { +; CHECK-LABEL: define noundef i64 @test_trunc_with_constexpr( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[ADD:%.*]] = add i32 trunc (i64 sub (i64 0, i64 ptrtoint (ptr @g_var to i64)) to i32), 1 +; CHECK-NEXT: [[CONV:%.*]] = sext i32 [[ADD]] to i64 +; CHECK-NEXT: ret i64 [[CONV]] +; + %add = add i32 trunc (i64 sub (i64 0, i64 ptrtoint (ptr @g_var to i64)) to i32), 1 + %conv = sext i32 %add to i64 + ret i64 %conv +} -- GitLab From 2d14ea68b8c0acdff7c040d581f7fde15d2683d9 Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Tue, 2 Apr 2024 10:22:03 +0200 Subject: [PATCH 0170/1690] [flang][NFC] speed-up external name conversion pass (#86814) The ExternalNameConversion pass can be surprisingly slow on big programs. On an example with a 50kloc Fortran file with about 10000 calls to external procedures, the pass alone took 25s on my machine. This patch reduces this to 0.16s. The root cause is that using `replaceAllSymbolUses` on each modified FuncOp is very expensive: it is walking all operations and attribute every time. An alternative would be to use mlir::SymbolUserMap to avoid walking the module again and again, but this is still much more expensive than what is needed because it is essentially caching all symbol uses of the module, and there is no need to such caching here. Instead: - Do a shallow walk of the module (only top level operation) to detect FuncOp/GlobalOp that needs to be updated. Update them and place the name remapping in a DenseMap. - If any remapping were done, do a single deep walk of the module operation, and update any SymbolRefAttr that matches a name that was remapped. --- .../Transforms/ExternalNameConversion.cpp | 155 ++++-------------- 1 file changed, 34 insertions(+), 121 deletions(-) diff --git a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp index 3a9686418c2e..b265c74c33dd 100644 --- a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp +++ b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp @@ -12,13 +12,9 @@ #include "flang/Optimizer/Dialect/FIROpsSupport.h" #include "flang/Optimizer/Support/InternalNames.h" #include "flang/Optimizer/Transforms/Passes.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/OpenACC/OpenACC.h" -#include "mlir/Dialect/OpenMP/OpenMPDialect.h" #include "mlir/IR/Attributes.h" #include "mlir/IR/SymbolTable.h" #include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" namespace fir { #define GEN_PASS_DEF_EXTERNALNAMECONVERSION @@ -44,102 +40,8 @@ mangleExternalName(const std::pair { -public: - using OpRewritePattern::OpRewritePattern; - - MangleNameOnFuncOp(mlir::MLIRContext *ctx, bool appendUnderscore) - : mlir::OpRewritePattern(ctx), - appendUnderscore(appendUnderscore) {} - - mlir::LogicalResult - matchAndRewrite(mlir::func::FuncOp op, - mlir::PatternRewriter &rewriter) const override { - mlir::LogicalResult ret = success(); - rewriter.startOpModification(op); - llvm::StringRef oldName = op.getSymName(); - auto result = fir::NameUniquer::deconstruct(oldName); - if (fir::NameUniquer::isExternalFacingUniquedName(result)) { - auto newSymbol = - rewriter.getStringAttr(mangleExternalName(result, appendUnderscore)); - - // Try to update all SymbolRef's in the module that match the current op - if (mlir::ModuleOp mod = op->getParentOfType()) - ret = op.replaceAllSymbolUses(newSymbol, mod); - - op.setSymNameAttr(newSymbol); - mlir::SymbolTable::setSymbolName(op, newSymbol); - - op->setAttr(fir::getInternalFuncNameAttrName(), - mlir::StringAttr::get(op->getContext(), oldName)); - } - rewriter.finalizeOpModification(op); - return ret; - } - -private: - bool appendUnderscore; -}; - -struct MangleNameForCommonBlock : public mlir::OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - MangleNameForCommonBlock(mlir::MLIRContext *ctx, bool appendUnderscore) - : mlir::OpRewritePattern(ctx), - appendUnderscore(appendUnderscore) {} - - mlir::LogicalResult - matchAndRewrite(fir::GlobalOp op, - mlir::PatternRewriter &rewriter) const override { - rewriter.startOpModification(op); - auto result = fir::NameUniquer::deconstruct( - op.getSymref().getRootReference().getValue()); - if (fir::NameUniquer::isExternalFacingUniquedName(result)) { - auto newName = mangleExternalName(result, appendUnderscore); - op.setSymrefAttr(mlir::SymbolRefAttr::get(op.getContext(), newName)); - SymbolTable::setSymbolName(op, newName); - } - rewriter.finalizeOpModification(op); - return success(); - } - -private: - bool appendUnderscore; -}; - -struct MangleNameOnAddrOfOp : public mlir::OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - MangleNameOnAddrOfOp(mlir::MLIRContext *ctx, bool appendUnderscore) - : mlir::OpRewritePattern(ctx), - appendUnderscore(appendUnderscore) {} - - mlir::LogicalResult - matchAndRewrite(fir::AddrOfOp op, - mlir::PatternRewriter &rewriter) const override { - auto result = fir::NameUniquer::deconstruct( - op.getSymbol().getRootReference().getValue()); - if (fir::NameUniquer::isExternalFacingUniquedName(result)) { - auto newName = SymbolRefAttr::get( - op.getContext(), mangleExternalName(result, appendUnderscore)); - rewriter.replaceOpWithNewOp(op, op.getResTy().getType(), - newName); - } - return success(); - } - -private: - bool appendUnderscore; -}; - class ExternalNameConversionPass : public fir::impl::ExternalNameConversionBase { public: @@ -162,31 +64,42 @@ void ExternalNameConversionPass::runOnOperation() { auto *context = &getContext(); appendUnderscores = (usePassOpt) ? appendUnderscoreOpt : appendUnderscores; + llvm::DenseMap remappings; + // Update names of external Fortran functions and names of Common Block + // globals. + for (auto &funcOrGlobal : op->getRegion(0).front()) { + if (llvm::isa(funcOrGlobal) || + llvm::isa(funcOrGlobal)) { + auto symName = funcOrGlobal.getAttrOfType( + mlir::SymbolTable::getSymbolAttrName()); + auto deconstructedName = fir::NameUniquer::deconstruct(symName); + if (fir::NameUniquer::isExternalFacingUniquedName(deconstructedName)) { + auto newName = mangleExternalName(deconstructedName, appendUnderscores); + auto newAttr = mlir::StringAttr::get(context, newName); + mlir::SymbolTable::setSymbolName(&funcOrGlobal, newAttr); + auto newSymRef = mlir::FlatSymbolRefAttr::get(newAttr); + remappings.try_emplace(symName, newSymRef); + if (llvm::isa(funcOrGlobal)) + funcOrGlobal.setAttr(fir::getInternalFuncNameAttrName(), symName); + } + } + } - mlir::RewritePatternSet patterns(context); - patterns.insert(context, appendUnderscores); - - ConversionTarget target(*context); - target.addLegalDialect(); - - target.addDynamicallyLegalOp([](mlir::func::FuncOp op) { - return !fir::NameUniquer::needExternalNameMangling(op.getSymName()); - }); - - target.addDynamicallyLegalOp([](fir::GlobalOp op) { - return !fir::NameUniquer::needExternalNameMangling( - op.getSymref().getRootReference().getValue()); - }); - - target.addDynamicallyLegalOp([](fir::AddrOfOp op) { - return !fir::NameUniquer::needExternalNameMangling( - op.getSymbol().getRootReference().getValue()); + if (remappings.empty()) + return; + + // Update all uses of the functions and globals that have been renamed. + op.walk([&remappings](mlir::Operation *nestedOp) { + llvm::SmallVector> updates; + for (const mlir::NamedAttribute &attr : nestedOp->getAttrDictionary()) + if (auto symRef = llvm::dyn_cast(attr.getValue())) + if (auto remap = remappings.find(symRef.getRootReference()); + remap != remappings.end()) + updates.emplace_back(std::pair{ + attr.getName(), mlir::SymbolRefAttr(remap->second)}); + for (auto update : updates) + nestedOp->setAttr(update.first, update.second); }); - - if (failed(applyPartialConversion(op, target, std::move(patterns)))) - signalPassFailure(); } std::unique_ptr fir::createExternalNameConversionPass() { -- GitLab From 77e5c0a95c54e0ca34b8e9c56c702490619b73c9 Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Tue, 2 Apr 2024 10:49:06 +0200 Subject: [PATCH 0171/1690] [AArch64][GISEL] Reduce likelihood of hash collisions for mappings in RegisterBankInfo (#87033) Fixes #85209 This patch removes the truncation from `hash_code` aka `size_t` down to `unsigned`, that currently happens on DenseMap accesses in RegisterBankInfo. This reduces the likelihood of hash collisions, as well as the likelihood of hitting EmptyKey or TombstoneKey, the special key values of DenseMap. This is not the ultimate solution to the problem, but we can do it in any case. --- llvm/include/llvm/CodeGen/RegisterBankInfo.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/include/llvm/CodeGen/RegisterBankInfo.h b/llvm/include/llvm/CodeGen/RegisterBankInfo.h index 62c4a57a605d..9704e3b1fded 100644 --- a/llvm/include/llvm/CodeGen/RegisterBankInfo.h +++ b/llvm/include/llvm/CodeGen/RegisterBankInfo.h @@ -399,22 +399,22 @@ protected: /// Keep dynamically allocated PartialMapping in a separate map. /// This shouldn't be needed when everything gets TableGen'ed. - mutable DenseMap> + mutable DenseMap> MapOfPartialMappings; /// Keep dynamically allocated ValueMapping in a separate map. /// This shouldn't be needed when everything gets TableGen'ed. - mutable DenseMap> + mutable DenseMap> MapOfValueMappings; /// Keep dynamically allocated array of ValueMapping in a separate map. /// This shouldn't be needed when everything gets TableGen'ed. - mutable DenseMap> + mutable DenseMap> MapOfOperandsMappings; /// Keep dynamically allocated InstructionMapping in a separate map. /// This shouldn't be needed when everything gets TableGen'ed. - mutable DenseMap> + mutable DenseMap> MapOfInstructionMappings; /// Getting the minimal register class of a physreg is expensive. -- GitLab From 6cce67a8f9bbab7ebaafa6f33e0efbb22dee3ea1 Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Tue, 2 Apr 2024 10:59:18 +0200 Subject: [PATCH 0172/1690] [SPIR-V] Fix validity of atomic instructions (#87051) This PR fixes validity of atomic instructions and improves type inference. More tests are able now to be accepted by `spirv-val`. --- llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp | 7 ++ llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 89 ++++++++++++++++--- llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 42 +++++++++ llvm/test/CodeGen/SPIRV/ExecutionMode.ll | 1 + .../test/CodeGen/SPIRV/instructions/atomic.ll | 28 ++++-- .../SPIRV/instructions/atomic_acqrel.ll | 28 ++++-- .../CodeGen/SPIRV/instructions/atomic_seq.ll | 28 ++++-- .../SPIRV/pointers/bitcast-fix-accesschain.ll | 37 ++++++++ .../pointers/type-deduce-by-call-complex.ll | 29 ++++++ 9 files changed, 252 insertions(+), 37 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-accesschain.ll create mode 100644 llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-complex.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp index ad4e72a3128b..1674cef7cb82 100644 --- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp @@ -251,6 +251,13 @@ static SPIRVType *getArgSPIRVType(const Function &F, unsigned ArgIdx, cast(II->getOperand(2))->getZExtValue(), ST)); } + // Replace PointerType with TypedPointerType to be able to map SPIR-V types to + // LLVM types in a consistent manner + if (isUntypedPointerTy(OriginalArgType)) { + OriginalArgType = + TypedPointerType::get(Type::getInt8Ty(F.getContext()), + getPointerAddressSpace(OriginalArgType)); + } return GR->getOrCreateSPIRVType(OriginalArgType, MIRBuilder, ArgAccessQual); } diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 7c5a38fa48d0..b341fcb41d03 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -65,6 +65,10 @@ class SPIRVEmitIntrinsics Type *deduceElementType(Value *I); Type *deduceElementTypeHelper(Value *I); Type *deduceElementTypeHelper(Value *I, std::unordered_set &Visited); + Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand, + std::unordered_set &Visited); + Type *deduceElementTypeByUsersDeep(Value *Op, + std::unordered_set &Visited); // deduce nested types of composites Type *deduceNestedTypeHelper(User *U); @@ -176,6 +180,44 @@ static inline void reportFatalOnTokenType(const Instruction *I) { false); } +// Set element pointer type to the given value of ValueTy and tries to +// specify this type further (recursively) by Operand value, if needed. +Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep( + Type *ValueTy, Value *Operand, std::unordered_set &Visited) { + Type *Ty = ValueTy; + if (Operand) { + if (auto *PtrTy = dyn_cast(Ty)) { + if (Type *NestedTy = deduceElementTypeHelper(Operand, Visited)) + Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace()); + } else { + Ty = deduceNestedTypeHelper(dyn_cast(Operand), Ty, Visited); + } + } + return Ty; +} + +// Traverse User instructions to deduce an element pointer type of the operand. +Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep( + Value *Op, std::unordered_set &Visited) { + if (!Op || !isPointerTy(Op->getType())) + return nullptr; + + if (auto PType = dyn_cast(Op->getType())) + return PType->getElementType(); + + // maybe we already know operand's element type + if (Type *KnownTy = GR->findDeducedElementType(Op)) + return KnownTy; + + for (User *OpU : Op->users()) { + if (Instruction *Inst = dyn_cast(OpU)) { + if (Type *Ty = deduceElementTypeHelper(Inst, Visited)) + return Ty; + } + } + return nullptr; +} + // Deduce and return a successfully deduced Type of the Instruction, // or nullptr otherwise. Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(Value *I) { @@ -206,21 +248,27 @@ Type *SPIRVEmitIntrinsics::deduceElementTypeHelper( } else if (auto *Ref = dyn_cast(I)) { Ty = Ref->getResultElementType(); } else if (auto *Ref = dyn_cast(I)) { - Ty = Ref->getValueType(); - if (Value *Op = Ref->getNumOperands() > 0 ? Ref->getOperand(0) : nullptr) { - if (auto *PtrTy = dyn_cast(Ty)) { - if (Type *NestedTy = deduceElementTypeHelper(Op, Visited)) - Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace()); - } else { - Ty = deduceNestedTypeHelper(dyn_cast(Op), Ty, Visited); - } - } + Ty = deduceElementTypeByValueDeep( + Ref->getValueType(), + Ref->getNumOperands() > 0 ? Ref->getOperand(0) : nullptr, Visited); } else if (auto *Ref = dyn_cast(I)) { Ty = deduceElementTypeHelper(Ref->getPointerOperand(), Visited); } else if (auto *Ref = dyn_cast(I)) { if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy(); isPointerTy(Src) && isPointerTy(Dest)) Ty = deduceElementTypeHelper(Ref->getOperand(0), Visited); + } else if (auto *Ref = dyn_cast(I)) { + Value *Op = Ref->getNewValOperand(); + Ty = deduceElementTypeByValueDeep(Op->getType(), Op, Visited); + } else if (auto *Ref = dyn_cast(I)) { + Value *Op = Ref->getValOperand(); + Ty = deduceElementTypeByValueDeep(Op->getType(), Op, Visited); + } else if (auto *Ref = dyn_cast(I)) { + for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) { + Ty = deduceElementTypeByUsersDeep(Ref->getIncomingValue(i), Visited); + if (Ty) + break; + } } // remember the found relationship @@ -293,6 +341,22 @@ Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper( return NewTy; } } + } else if (auto *VecTy = dyn_cast(OrigTy)) { + if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(0) : nullptr) { + Type *OpTy = VecTy->getElementType(); + Type *Ty = OpTy; + if (auto *PtrTy = dyn_cast(OpTy)) { + if (Type *NestedTy = deduceElementTypeHelper(Op, Visited)) + Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace()); + } else { + Ty = deduceNestedTypeHelper(dyn_cast(Op), OpTy, Visited); + } + if (Ty != OpTy) { + Type *NewTy = VectorType::get(Ty, VecTy->getElementCount()); + GR->addDeducedCompositeType(U, NewTy); + return NewTy; + } + } } return OrigTy; @@ -578,7 +642,8 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, // Handle calls to builtins (non-intrinsics): CallInst *CI = dyn_cast(I); - if (!CI || CI->isIndirectCall() || CI->getCalledFunction()->isIntrinsic()) + if (!CI || CI->isIndirectCall() || CI->isInlineAsm() || + !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic()) return; // collect information about formal parameter types @@ -929,6 +994,10 @@ Type *SPIRVEmitIntrinsics::deduceFunParamElementType( // maybe we already know operand's element type if (Type *KnownTy = GR->findDeducedElementType(OpArg)) return KnownTy; + // try to deduce from the operand itself + Visited.clear(); + if (Type *Ty = deduceElementTypeHelper(OpArg, Visited)) + return Ty; // search in actual parameter's users for (User *OpU : OpArg->users()) { Instruction *Inst = dyn_cast(OpU); diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp index 4f5c1dc4f90b..90a31551f45a 100644 --- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp @@ -201,6 +201,17 @@ void validateForwardCalls(const SPIRVSubtarget &STI, } } +// Validation of an access chain. +void validateAccessChain(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, + SPIRVGlobalRegistry &GR, MachineInstr &I) { + SPIRVType *BaseTypeInst = GR.getSPIRVTypeForVReg(I.getOperand(0).getReg()); + if (BaseTypeInst && BaseTypeInst->getOpcode() == SPIRV::OpTypePointer) { + SPIRVType *BaseElemType = + GR.getSPIRVTypeForVReg(BaseTypeInst->getOperand(2).getReg()); + validatePtrTypes(STI, MRI, GR, I, 2, BaseElemType); + } +} + // TODO: the logic of inserting additional bitcast's is to be moved // to pre-IRTranslation passes eventually void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const { @@ -213,16 +224,47 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const { MBBI != MBBE;) { MachineInstr &MI = *MBBI++; switch (MI.getOpcode()) { + case SPIRV::OpAtomicLoad: + case SPIRV::OpAtomicExchange: + case SPIRV::OpAtomicCompareExchange: + case SPIRV::OpAtomicCompareExchangeWeak: + case SPIRV::OpAtomicIIncrement: + case SPIRV::OpAtomicIDecrement: + case SPIRV::OpAtomicIAdd: + case SPIRV::OpAtomicISub: + case SPIRV::OpAtomicSMin: + case SPIRV::OpAtomicUMin: + case SPIRV::OpAtomicSMax: + case SPIRV::OpAtomicUMax: + case SPIRV::OpAtomicAnd: + case SPIRV::OpAtomicOr: + case SPIRV::OpAtomicXor: + // for the above listed instructions + // OpAtomicXXX , ptr %Op, ... + // implies that %Op is a pointer to case SPIRV::OpLoad: // OpLoad , ptr %Op implies that %Op is a pointer to validatePtrTypes(STI, MRI, GR, MI, 2, GR.getSPIRVTypeForVReg(MI.getOperand(0).getReg())); break; + case SPIRV::OpAtomicStore: + // OpAtomicStore ptr %Op, , , + // implies that %Op points to the 's type + validatePtrTypes(STI, MRI, GR, MI, 0, + GR.getSPIRVTypeForVReg(MI.getOperand(3).getReg())); + break; case SPIRV::OpStore: // OpStore ptr %Op, implies that %Op points to the 's type validatePtrTypes(STI, MRI, GR, MI, 0, GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg())); break; + case SPIRV::OpPtrCastToGeneric: + validateAccessChain(STI, MRI, GR, MI); + break; + case SPIRV::OpInBoundsPtrAccessChain: + if (MI.getNumOperands() == 4) + validateAccessChain(STI, MRI, GR, MI); + break; case SPIRV::OpFunctionCall: // ensure there is no mismatch between actual and expected arg types: diff --git a/llvm/test/CodeGen/SPIRV/ExecutionMode.ll b/llvm/test/CodeGen/SPIRV/ExecutionMode.ll index 3e321e1c2bd2..180b7246952d 100644 --- a/llvm/test/CodeGen/SPIRV/ExecutionMode.ll +++ b/llvm/test/CodeGen/SPIRV/ExecutionMode.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: %[[#VOID:]] = OpTypeVoid diff --git a/llvm/test/CodeGen/SPIRV/instructions/atomic.ll b/llvm/test/CodeGen/SPIRV/instructions/atomic.ll index 9715504fcc5d..ce59bb206402 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/atomic.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/atomic.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: OpName [[ADD:%.*]] "test_add" ; CHECK-DAG: OpName [[SUB:%.*]] "test_sub" @@ -20,7 +21,8 @@ ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicIAdd [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicIAdd [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_add(i32* %ptr, i32 %val) { @@ -32,7 +34,8 @@ define i32 @test_add(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicISub [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicISub [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_sub(i32* %ptr, i32 %val) { @@ -44,7 +47,8 @@ define i32 @test_sub(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicSMin [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicSMin [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_min(i32* %ptr, i32 %val) { @@ -56,7 +60,8 @@ define i32 @test_min(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicSMax [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicSMax [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_max(i32* %ptr, i32 %val) { @@ -68,7 +73,8 @@ define i32 @test_max(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicUMin [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicUMin [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_umin(i32* %ptr, i32 %val) { @@ -80,7 +86,8 @@ define i32 @test_umin(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicUMax [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicUMax [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_umax(i32* %ptr, i32 %val) { @@ -92,7 +99,8 @@ define i32 @test_umax(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicAnd [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicAnd [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_and(i32* %ptr, i32 %val) { @@ -104,7 +112,8 @@ define i32 @test_and(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicOr [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicOr [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_or(i32* %ptr, i32 %val) { @@ -116,7 +125,8 @@ define i32 @test_or(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicXor [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicXor [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_xor(i32* %ptr, i32 %val) { diff --git a/llvm/test/CodeGen/SPIRV/instructions/atomic_acqrel.ll b/llvm/test/CodeGen/SPIRV/instructions/atomic_acqrel.ll index 63c0ae75f5ec..950dfe417637 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/atomic_acqrel.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/atomic_acqrel.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: OpName [[ADD:%.*]] "test_add" ; CHECK-DAG: OpName [[SUB:%.*]] "test_sub" @@ -20,7 +21,8 @@ ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicIAdd [[I32Ty]] [[A]] [[SCOPE]] [[ACQREL]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicIAdd [[I32Ty]] [[BC_A]] [[SCOPE]] [[ACQREL]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_add(i32* %ptr, i32 %val) { @@ -32,7 +34,8 @@ define i32 @test_add(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicISub [[I32Ty]] [[A]] [[SCOPE]] [[ACQREL]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicISub [[I32Ty]] [[BC_A]] [[SCOPE]] [[ACQREL]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_sub(i32* %ptr, i32 %val) { @@ -44,7 +47,8 @@ define i32 @test_sub(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicSMin [[I32Ty]] [[A]] [[SCOPE]] [[ACQREL]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicSMin [[I32Ty]] [[BC_A]] [[SCOPE]] [[ACQREL]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_min(i32* %ptr, i32 %val) { @@ -56,7 +60,8 @@ define i32 @test_min(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicSMax [[I32Ty]] [[A]] [[SCOPE]] [[ACQREL]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicSMax [[I32Ty]] [[BC_A]] [[SCOPE]] [[ACQREL]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_max(i32* %ptr, i32 %val) { @@ -68,7 +73,8 @@ define i32 @test_max(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicUMin [[I32Ty]] [[A]] [[SCOPE]] [[ACQREL]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicUMin [[I32Ty]] [[BC_A]] [[SCOPE]] [[ACQREL]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_umin(i32* %ptr, i32 %val) { @@ -80,7 +86,8 @@ define i32 @test_umin(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicUMax [[I32Ty]] [[A]] [[SCOPE]] [[ACQREL]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicUMax [[I32Ty]] [[BC_A]] [[SCOPE]] [[ACQREL]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_umax(i32* %ptr, i32 %val) { @@ -92,7 +99,8 @@ define i32 @test_umax(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicAnd [[I32Ty]] [[A]] [[SCOPE]] [[ACQREL]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicAnd [[I32Ty]] [[BC_A]] [[SCOPE]] [[ACQREL]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_and(i32* %ptr, i32 %val) { @@ -104,7 +112,8 @@ define i32 @test_and(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicOr [[I32Ty]] [[A]] [[SCOPE]] [[ACQREL]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicOr [[I32Ty]] [[BC_A]] [[SCOPE]] [[ACQREL]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_or(i32* %ptr, i32 %val) { @@ -116,7 +125,8 @@ define i32 @test_or(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicXor [[I32Ty]] [[A]] [[SCOPE]] [[ACQREL]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicXor [[I32Ty]] [[BC_A]] [[SCOPE]] [[ACQREL]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_xor(i32* %ptr, i32 %val) { diff --git a/llvm/test/CodeGen/SPIRV/instructions/atomic_seq.ll b/llvm/test/CodeGen/SPIRV/instructions/atomic_seq.ll index f6a8fe1e6db1..f142e012dcb7 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/atomic_seq.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/atomic_seq.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: OpName [[ADD:%.*]] "test_add" ; CHECK-DAG: OpName [[SUB:%.*]] "test_sub" @@ -20,7 +21,8 @@ ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicIAdd [[I32Ty]] [[A]] [[SCOPE]] [[SEQ]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicIAdd [[I32Ty]] [[BC_A]] [[SCOPE]] [[SEQ]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_add(i32* %ptr, i32 %val) { @@ -32,7 +34,8 @@ define i32 @test_add(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicISub [[I32Ty]] [[A]] [[SCOPE]] [[SEQ]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicISub [[I32Ty]] [[BC_A]] [[SCOPE]] [[SEQ]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_sub(i32* %ptr, i32 %val) { @@ -44,7 +47,8 @@ define i32 @test_sub(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicSMin [[I32Ty]] [[A]] [[SCOPE]] [[SEQ]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicSMin [[I32Ty]] [[BC_A]] [[SCOPE]] [[SEQ]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_min(i32* %ptr, i32 %val) { @@ -56,7 +60,8 @@ define i32 @test_min(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicSMax [[I32Ty]] [[A]] [[SCOPE]] [[SEQ]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicSMax [[I32Ty]] [[BC_A]] [[SCOPE]] [[SEQ]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_max(i32* %ptr, i32 %val) { @@ -68,7 +73,8 @@ define i32 @test_max(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicUMin [[I32Ty]] [[A]] [[SCOPE]] [[SEQ]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicUMin [[I32Ty]] [[BC_A]] [[SCOPE]] [[SEQ]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_umin(i32* %ptr, i32 %val) { @@ -80,7 +86,8 @@ define i32 @test_umin(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicUMax [[I32Ty]] [[A]] [[SCOPE]] [[SEQ]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicUMax [[I32Ty]] [[BC_A]] [[SCOPE]] [[SEQ]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_umax(i32* %ptr, i32 %val) { @@ -92,7 +99,8 @@ define i32 @test_umax(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicAnd [[I32Ty]] [[A]] [[SCOPE]] [[SEQ]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicAnd [[I32Ty]] [[BC_A]] [[SCOPE]] [[SEQ]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_and(i32* %ptr, i32 %val) { @@ -104,7 +112,8 @@ define i32 @test_and(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicOr [[I32Ty]] [[A]] [[SCOPE]] [[SEQ]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicOr [[I32Ty]] [[BC_A]] [[SCOPE]] [[SEQ]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_or(i32* %ptr, i32 %val) { @@ -116,7 +125,8 @@ define i32 @test_or(i32* %ptr, i32 %val) { ; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter ; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter ; CHECK-NEXT: OpLabel -; CHECK-NEXT: [[R:%.*]] = OpAtomicXor [[I32Ty]] [[A]] [[SCOPE]] [[SEQ]] [[B]] +; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]] +; CHECK-NEXT: [[R:%.*]] = OpAtomicXor [[I32Ty]] [[BC_A]] [[SCOPE]] [[SEQ]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd define i32 @test_xor(i32* %ptr, i32 %val) { diff --git a/llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-accesschain.ll b/llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-accesschain.ll new file mode 100644 index 000000000000..7fae6ca2c48c --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-accesschain.ll @@ -0,0 +1,37 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: %[[#TYCHAR:]] = OpTypeInt 8 0 +; CHECK-DAG: %[[#TYCHARPTR:]] = OpTypePointer Function %[[#TYCHAR]] +; CHECK-DAG: %[[#TYINT32:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#TYSTRUCTINT32:]] = OpTypeStruct %[[#TYINT32]] +; CHECK-DAG: %[[#TYARRAY:]] = OpTypeArray %[[#TYSTRUCTINT32]] %[[#]] +; CHECK-DAG: %[[#TYSTRUCT:]] = OpTypeStruct %[[#TYARRAY]] +; CHECK-DAG: %[[#TYSTRUCTPTR:]] = OpTypePointer Function %[[#TYSTRUCT]] +; CHECK-DAG: %[[#TYINT64:]] = OpTypeInt 64 0 +; CHECK-DAG: %[[#TYINT64PTR:]] = OpTypePointer Function %[[#TYINT64]] +; CHECK: OpFunction +; CHECK: %[[#PTRTOSTRUCT:]] = OpFunctionParameter %[[#TYSTRUCTPTR]] +; CHECK: %[[#PTRTOCHAR:]] = OpBitcast %[[#TYCHARPTR]] %[[#PTRTOSTRUCT]] +; CHECK-NEXT: OpInBoundsPtrAccessChain %[[#TYCHARPTR]] %[[#PTRTOCHAR]] +; CHECK: OpFunction +; CHECK: %[[#PTRTOSTRUCT2:]] = OpFunctionParameter %[[#TYSTRUCTPTR]] +; CHECK: %[[#ELEM:]] = OpInBoundsPtrAccessChain %[[#TYSTRUCTPTR]] %[[#PTRTOSTRUCT2]] +; CHECK-NEXT: %[[#TOLOAD:]] = OpBitcast %[[#TYINT64PTR]] %[[#ELEM]] +; CHECK-NEXT: OpLoad %[[#TYINT64]] %[[#TOLOAD]] + +%struct.S = type { i32 } +%struct.__wrapper_class = type { [7 x %struct.S] } + +define spir_kernel void @foo1(ptr noundef byval(%struct.__wrapper_class) align 4 %_arg_Arr) { +entry: + %elem = getelementptr inbounds i8, ptr %_arg_Arr, i64 0 + ret void +} + +define spir_kernel void @foo2(ptr noundef byval(%struct.__wrapper_class) align 4 %_arg_Arr) { +entry: + %elem = getelementptr inbounds %struct.__wrapper_class, ptr %_arg_Arr, i64 0 + %data = load i64, ptr %elem + ret void +} diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-complex.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-complex.ll new file mode 100644 index 000000000000..ea7a22c31d0e --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-complex.ll @@ -0,0 +1,29 @@ +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-SPIRV-DAG: %[[Long:.*]] = OpTypeInt 32 0 +; CHECK-SPIRV-DAG: %[[Void:.*]] = OpTypeVoid +; CHECK-SPIRV-DAG: %[[Struct:.*]] = OpTypeStruct %[[Long]] +; CHECK-SPIRV-DAG: %[[StructPtr:.*]] = OpTypePointer Generic %[[Struct]] +; CHECK-SPIRV-DAG: %[[Function:.*]] = OpTypeFunction %[[Void]] %[[StructPtr]] +; CHECK-SPIRV-DAG: %[[Const:.*]] = OpConstantNull %[[Struct]] +; CHECK-SPIRV-DAG: %[[CrossStructPtr:.*]] = OpTypePointer CrossWorkgroup %[[Struct]] +; CHECK-SPIRV-DAG: %[[Var:.*]] = OpVariable %[[CrossStructPtr]] CrossWorkgroup %[[Const]] +; CHECK-SPIRV: %[[Foo:.*]] = OpFunction %[[Void]] None %[[Function]] +; CHECK-SPIRV-NEXT: OpFunctionParameter %[[StructPtr]] +; CHECK-SPIRV: %[[Casted:.*]] = OpPtrCastToGeneric %[[StructPtr]] %[[Var]] +; CHECK-SPIRV-NEXT: OpFunctionCall %[[Void]] %[[Foo]] %[[Casted]] + +%struct.global_ctor_dtor = type { i32 } +@g1 = addrspace(1) global %struct.global_ctor_dtor zeroinitializer + +define linkonce_odr spir_func void @foo(ptr addrspace(4) %this) { +entry: + ret void +} + +define internal spir_func void @bar() { +entry: + call spir_func void @foo(ptr addrspace(4) addrspacecast (ptr addrspace(1) @g1 to ptr addrspace(4))) + ret void +} -- GitLab From 6654235594d86e7ed70abb7358ed25029d1560e5 Mon Sep 17 00:00:00 2001 From: Sizov Nikita Date: Tue, 2 Apr 2024 12:39:49 +0300 Subject: [PATCH 0173/1690] [SelectionDAG] implement computeKnownBits for add AVG* instructions (#86754) knownBits calculation for **AVGFLOORU** / **AVGFLOORS** / **AVGCEILU** / **AVGCEILS** instructions Prerequisite for #76644 --- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 15 ++++-- .../CodeGen/AArch64SelectionDAGTest.cpp | 48 +++++++++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index e8d1ac1d3a91..e3b76b95eb86 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -3419,13 +3419,18 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known = KnownBits::mulhs(Known, Known2); break; } - case ISD::AVGCEILU: { + case ISD::AVGFLOORU: + case ISD::AVGCEILU: + case ISD::AVGFLOORS: + case ISD::AVGCEILS: { + bool IsCeil = Opcode == ISD::AVGCEILU || Opcode == ISD::AVGCEILS; + bool IsSigned = Opcode == ISD::AVGFLOORS || Opcode == ISD::AVGCEILS; Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - Known = Known.zext(BitWidth + 1); - Known2 = Known2.zext(BitWidth + 1); - KnownBits One = KnownBits::makeConstant(APInt(1, 1)); - Known = KnownBits::computeForAddCarry(Known, Known2, One); + Known = IsSigned ? Known.sext(BitWidth + 1) : Known.zext(BitWidth + 1); + Known2 = IsSigned ? Known2.sext(BitWidth + 1) : Known2.zext(BitWidth + 1); + KnownBits Carry = KnownBits::makeConstant(APInt(1, IsCeil ? 1 : 0)); + Known = KnownBits::computeForAddCarry(Known, Known2, Carry); Known = Known.extractBits(BitWidth, 1); break; } diff --git a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp index e0772684e3a9..27bcad7c24c4 100644 --- a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp +++ b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp @@ -796,4 +796,52 @@ TEST_F(AArch64SelectionDAGTest, computeKnownBits_extload_knownnegative) { EXPECT_EQ(Known.One, APInt(32, 0xfffffff0)); } +TEST_F(AArch64SelectionDAGTest, + computeKnownBits_AVGFLOORU_AVGFLOORS_AVGCEILU_AVGCEILS) { + SDLoc Loc; + auto Int8VT = EVT::getIntegerVT(Context, 8); + auto Int16VT = EVT::getIntegerVT(Context, 16); + auto Int8Vec8VT = EVT::getVectorVT(Context, Int8VT, 8); + auto Int16Vec8VT = EVT::getVectorVT(Context, Int16VT, 8); + + SDValue UnknownOp0 = DAG->getRegister(0, Int8Vec8VT); + SDValue UnknownOp1 = DAG->getRegister(1, Int8Vec8VT); + + SDValue ZextOp0 = + DAG->getNode(ISD::ZERO_EXTEND, Loc, Int16Vec8VT, UnknownOp0); + SDValue ZextOp1 = + DAG->getNode(ISD::ZERO_EXTEND, Loc, Int16Vec8VT, UnknownOp1); + // ZextOp0 = 00000000???????? + // ZextOp1 = 00000000???????? + // => (for all AVG* instructions) + // Known.Zero = 1111111100000000 (0xFF00) + // Known.One = 0000000000000000 (0x0000) + auto Zeroes = APInt(16, 0xFF00); + auto Ones = APInt(16, 0x0000); + + SDValue AVGFLOORU = + DAG->getNode(ISD::AVGFLOORU, Loc, Int16Vec8VT, ZextOp0, ZextOp1); + KnownBits KnownAVGFLOORU = DAG->computeKnownBits(AVGFLOORU); + EXPECT_EQ(KnownAVGFLOORU.Zero, Zeroes); + EXPECT_EQ(KnownAVGFLOORU.One, Ones); + + SDValue AVGFLOORS = + DAG->getNode(ISD::AVGFLOORU, Loc, Int16Vec8VT, ZextOp0, ZextOp1); + KnownBits KnownAVGFLOORS = DAG->computeKnownBits(AVGFLOORS); + EXPECT_EQ(KnownAVGFLOORS.Zero, Zeroes); + EXPECT_EQ(KnownAVGFLOORS.One, Ones); + + SDValue AVGCEILU = + DAG->getNode(ISD::AVGCEILU, Loc, Int16Vec8VT, ZextOp0, ZextOp1); + KnownBits KnownAVGCEILU = DAG->computeKnownBits(AVGCEILU); + EXPECT_EQ(KnownAVGCEILU.Zero, Zeroes); + EXPECT_EQ(KnownAVGCEILU.One, Ones); + + SDValue AVGCEILS = + DAG->getNode(ISD::AVGCEILS, Loc, Int16Vec8VT, ZextOp0, ZextOp1); + KnownBits KnownAVGCEILS = DAG->computeKnownBits(AVGCEILS); + EXPECT_EQ(KnownAVGCEILS.Zero, Zeroes); + EXPECT_EQ(KnownAVGCEILS.One, Ones); +} + } // end namespace llvm -- GitLab From 89cfae41ecc043f8c47be4dea4b7c740d4f950b3 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Tue, 2 Apr 2024 11:37:20 +0200 Subject: [PATCH 0174/1690] [mlir] Add missing #include header for std::is_pointer --- mlir/include/mlir/IR/OwningOpRef.h | 1 + 1 file changed, 1 insertion(+) diff --git a/mlir/include/mlir/IR/OwningOpRef.h b/mlir/include/mlir/IR/OwningOpRef.h index eb4bf2dc67e3..0c35eae8de09 100644 --- a/mlir/include/mlir/IR/OwningOpRef.h +++ b/mlir/include/mlir/IR/OwningOpRef.h @@ -13,6 +13,7 @@ #ifndef MLIR_IR_OWNINGOPREF_H #define MLIR_IR_OWNINGOPREF_H +#include #include namespace mlir { -- GitLab From 16da9d53519214475c04109d953022f272ac8022 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 2 Apr 2024 10:43:34 +0100 Subject: [PATCH 0175/1690] [VPlan] Remove redundant set of debug loc in VPInstruction (NFCI). Consistently use setDebugLocFrom and remove redundant setDebugLocFrom. --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 23d025cf33ea..124ae3108d8a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -455,8 +455,6 @@ Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) { RecurKind RK = RdxDesc.getRecurrenceKind(); - State.setDebugLocFrom(getDebugLoc()); - VPValue *LoopExitingDef = getOperand(1); Type *PhiTy = OrigPhi->getType(); VectorParts RdxParts(State.UF); @@ -551,7 +549,7 @@ void VPInstruction::execute(VPTransformState &State) { "Recipe not a FPMathOp but has fast-math flags?"); if (hasFastMathFlags()) State.Builder.setFastMathFlags(getFastMathFlags()); - State.Builder.SetCurrentDebugLocation(getDebugLoc()); + State.setDebugLocFrom(getDebugLoc()); bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() && (vputils::onlyFirstLaneUsed(this) || -- GitLab From 46246683a61a081d9d78cf987fd4f024556ecdc8 Mon Sep 17 00:00:00 2001 From: Rin Dobrescu Date: Tue, 2 Apr 2024 10:47:51 +0100 Subject: [PATCH 0176/1690] [AArch64] Update Neoverse V2 FSQRT execution units in schedule model. (#86803) This patch updates the SVE FSQRT instruction execution units to be able to run on VX0 and VX2. --- llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td | 10 ++++------ .../AArch64/Neoverse/V2-sve-instructions.s | 14 +++++++------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td index f10b94523d2e..4d7f44e7b9b9 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td +++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td @@ -1076,14 +1076,12 @@ def V2Write_7cyc_1V02_7rc : SchedWriteRes<[V2UnitV02]> { let Latency = 7; let def V2Write_10cyc_1V02_5rc : SchedWriteRes<[V2UnitV02]> { let Latency = 10; let ReleaseAtCycles = [ 5]; } def V2Write_10cyc_1V02_9rc : SchedWriteRes<[V2UnitV02]> { let Latency = 10; let ReleaseAtCycles = [ 9]; } def V2Write_10cyc_1V02_10rc : SchedWriteRes<[V2UnitV02]> { let Latency = 10; let ReleaseAtCycles = [10]; } -def V2Write_10cyc_1V0_9rc : SchedWriteRes<[V2UnitV0]> { let Latency = 10; let ReleaseAtCycles = [ 9]; } def V2Write_10cyc_1V1_9rc : SchedWriteRes<[V2UnitV1]> { let Latency = 10; let ReleaseAtCycles = [ 9]; } -def V2Write_13cyc_1V0_12rc : SchedWriteRes<[V2UnitV0]> { let Latency = 13; let ReleaseAtCycles = [12]; } def V2Write_13cyc_1V02_12rc : SchedWriteRes<[V2UnitV02]> { let Latency = 13; let ReleaseAtCycles = [12]; } def V2Write_13cyc_1V02_13rc : SchedWriteRes<[V2UnitV02]> { let Latency = 13; let ReleaseAtCycles = [13]; } def V2Write_15cyc_1V02_14rc : SchedWriteRes<[V2UnitV02]> { let Latency = 15; let ReleaseAtCycles = [14]; } +def V2Write_16cyc_1V02_14rc : SchedWriteRes<[V2UnitV02]> { let Latency = 16; let ReleaseAtCycles = [14]; } def V2Write_16cyc_1V02_15rc : SchedWriteRes<[V2UnitV02]> { let Latency = 16; let ReleaseAtCycles = [15]; } -def V2Write_16cyc_1V0_14rc : SchedWriteRes<[V2UnitV0]> { let Latency = 16; let ReleaseAtCycles = [14]; } // Miscellaneous // ----------------------------------------------------------------------------- @@ -2567,13 +2565,13 @@ def : InstRW<[V2Write_4cyc_2V02], (instregex "^FRINT[AIMNPXZ]_ZPmZ_S")>; def : InstRW<[V2Write_3cyc_1V02], (instregex "^FRINT[AIMNPXZ]_ZPmZ_D")>; // Floating point square root, F16 -def : InstRW<[V2Write_13cyc_1V0_12rc], (instregex "^FSQRT_ZPmZ_H")>; +def : InstRW<[V2Write_13cyc_1V02_12rc], (instregex "^FSQRT_ZPmZ_H")>; // Floating point square root, F32 -def : InstRW<[V2Write_10cyc_1V0_9rc], (instregex "^FSQRT_ZPmZ_S")>; +def : InstRW<[V2Write_10cyc_1V02_9rc], (instregex "^FSQRT_ZPmZ_S")>; // Floating point square root, F64 -def : InstRW<[V2Write_16cyc_1V0_14rc], (instregex "^FSQRT_ZPmZ_D")>; +def : InstRW<[V2Write_16cyc_1V02_14rc], (instregex "^FSQRT_ZPmZ_D")>; // Floating point trigonometric exponentiation def : InstRW<[V2Write_3cyc_1V1], (instregex "^FEXPA_ZZ_[HSD]")>; diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s index 4d6ce706b052..acd355682494 100644 --- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s +++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s @@ -4278,9 +4278,9 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.25 fscale z0.d, p7/m, z0.d, z31.d # CHECK-NEXT: 1 3 0.25 fscale z0.h, p7/m, z0.h, z31.h # CHECK-NEXT: 1 3 0.25 fscale z0.s, p7/m, z0.s, z31.s -# CHECK-NEXT: 1 16 14.00 fsqrt z31.d, p7/m, z31.d -# CHECK-NEXT: 1 13 12.00 fsqrt z31.h, p7/m, z31.h -# CHECK-NEXT: 1 10 9.00 fsqrt z31.s, p7/m, z31.s +# CHECK-NEXT: 1 16 7.00 fsqrt z31.d, p7/m, z31.d +# CHECK-NEXT: 1 13 6.00 fsqrt z31.h, p7/m, z31.h +# CHECK-NEXT: 1 10 4.50 fsqrt z31.s, p7/m, z31.s # CHECK-NEXT: 1 2 0.25 fsub z0.d, p0/m, z0.d, #0.5 # CHECK-NEXT: 1 2 0.25 fsub z0.d, p7/m, z0.d, z31.d # CHECK-NEXT: 1 2 0.25 fsub z0.d, z1.d, z31.d @@ -6861,7 +6861,7 @@ zip2 z31.s, z31.s, z31.s # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0.0] [0.1] [1.0] [1.1] [2] [3.0] [3.1] [4] [5] [6] [7] [8] [9] [10] [11] [12] [13] -# CHECK-NEXT: - - - - 245.00 651.00 651.00 570.50 272.50 83.75 83.75 81.75 81.75 1554.25 1281.75 776.75 748.25 +# CHECK-NEXT: - - - - 245.00 651.00 651.00 570.50 272.50 83.75 83.75 81.75 81.75 1536.75 1281.75 794.25 748.25 # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0.0] [0.1] [1.0] [1.1] [2] [3.0] [3.1] [4] [5] [6] [7] [8] [9] [10] [11] [12] [13] Instructions: @@ -7718,9 +7718,9 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - - - - - - - - 0.25 0.25 0.25 0.25 fscale z0.d, p7/m, z0.d, z31.d # CHECK-NEXT: - - - - - - - - - - - - - 0.25 0.25 0.25 0.25 fscale z0.h, p7/m, z0.h, z31.h # CHECK-NEXT: - - - - - - - - - - - - - 0.25 0.25 0.25 0.25 fscale z0.s, p7/m, z0.s, z31.s -# CHECK-NEXT: - - - - - - - - - - - - - 14.00 - - - fsqrt z31.d, p7/m, z31.d -# CHECK-NEXT: - - - - - - - - - - - - - 12.00 - - - fsqrt z31.h, p7/m, z31.h -# CHECK-NEXT: - - - - - - - - - - - - - 9.00 - - - fsqrt z31.s, p7/m, z31.s +# CHECK-NEXT: - - - - - - - - - - - - - 7.00 - 7.00 - fsqrt z31.d, p7/m, z31.d +# CHECK-NEXT: - - - - - - - - - - - - - 6.00 - 6.00 - fsqrt z31.h, p7/m, z31.h +# CHECK-NEXT: - - - - - - - - - - - - - 4.50 - 4.50 - fsqrt z31.s, p7/m, z31.s # CHECK-NEXT: - - - - - - - - - - - - - 0.25 0.25 0.25 0.25 fsub z0.d, p0/m, z0.d, #0.5 # CHECK-NEXT: - - - - - - - - - - - - - 0.25 0.25 0.25 0.25 fsub z0.d, p7/m, z0.d, z31.d # CHECK-NEXT: - - - - - - - - - - - - - 0.25 0.25 0.25 0.25 fsub z0.d, z1.d, z31.d -- GitLab From 1d06f41b72e429a5b3ba318ff639b8b997e21ff8 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Tue, 2 Apr 2024 10:58:45 +0100 Subject: [PATCH 0177/1690] [VectorCombine] foldBitcastShuffle - peek through any residual bitcasts before creating a new bitcast on top (#86119) Encountered while working on #67803, wading through the chains of bitcasts that SSE intrinsics introduces - this patch helps prevents cases where the bitcast chains aren't cleared out and we can't perform further combines until after InstCombine/InstSimplify has run. --- llvm/lib/Transforms/Vectorize/VectorCombine.cpp | 12 ++++++++++-- .../VectorCombine/X86/shuffle-inseltpoison.ll | 6 ++---- llvm/test/Transforms/VectorCombine/X86/shuffle.ll | 6 ++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp index 7e86137f23f3..af5e7c9bc385 100644 --- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp +++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp @@ -135,6 +135,14 @@ private: }; } // namespace +/// Return the source operand of a potentially bitcasted value. If there is no +/// bitcast, return the input value itself. +static Value *peekThroughBitcasts(Value *V) { + while (auto *BitCast = dyn_cast(V)) + V = BitCast->getOperand(0); + return V; +} + static bool canWidenLoad(LoadInst *Load, const TargetTransformInfo &TTI) { // Do not widen load if atomic/volatile or under asan/hwasan/memtag/tsan. // The widened load may load data from dirty regions or create data races @@ -751,8 +759,8 @@ bool VectorCombine::foldBitcastShuffle(Instruction &I) { // bitcast (shuf V0, V1, MaskC) --> shuf (bitcast V0), (bitcast V1), MaskC' ++NumShufOfBitcast; - Value *CastV0 = Builder.CreateBitCast(V0, NewShuffleTy); - Value *CastV1 = Builder.CreateBitCast(V1, NewShuffleTy); + Value *CastV0 = Builder.CreateBitCast(peekThroughBitcasts(V0), NewShuffleTy); + Value *CastV1 = Builder.CreateBitCast(peekThroughBitcasts(V1), NewShuffleTy); Value *Shuf = Builder.CreateShuffleVector(CastV0, CastV1, NewMask); replaceValue(I, *Shuf); return true; diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-inseltpoison.ll index 8c5c6656ca17..74a58c8d3136 100644 --- a/llvm/test/Transforms/VectorCombine/X86/shuffle-inseltpoison.ll +++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-inseltpoison.ll @@ -133,8 +133,7 @@ define <2 x i64> @PR35454_1(<2 x i64> %v) { ; SSE-NEXT: ret <2 x i64> [[BC3]] ; ; AVX-LABEL: @PR35454_1( -; AVX-NEXT: [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32> -; AVX-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[BC]] to <16 x i8> +; AVX-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[V:%.*]] to <16 x i8> ; AVX-NEXT: [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> ; AVX-NEXT: [[ADD:%.*]] = shl <16 x i8> [[BC1]], ; AVX-NEXT: [[BC2:%.*]] = bitcast <16 x i8> [[ADD]] to <4 x i32> @@ -164,8 +163,7 @@ define <2 x i64> @PR35454_2(<2 x i64> %v) { ; SSE-NEXT: ret <2 x i64> [[BC3]] ; ; AVX-LABEL: @PR35454_2( -; AVX-NEXT: [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32> -; AVX-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[BC]] to <8 x i16> +; AVX-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[V:%.*]] to <8 x i16> ; AVX-NEXT: [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> ; AVX-NEXT: [[ADD:%.*]] = shl <8 x i16> [[BC1]], ; AVX-NEXT: [[BC2:%.*]] = bitcast <8 x i16> [[ADD]] to <4 x i32> diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll index 60cfc4d4b070..d1484fd5ab33 100644 --- a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll +++ b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll @@ -133,8 +133,7 @@ define <2 x i64> @PR35454_1(<2 x i64> %v) { ; SSE-NEXT: ret <2 x i64> [[BC3]] ; ; AVX-LABEL: @PR35454_1( -; AVX-NEXT: [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32> -; AVX-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[BC]] to <16 x i8> +; AVX-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[V:%.*]] to <16 x i8> ; AVX-NEXT: [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> ; AVX-NEXT: [[ADD:%.*]] = shl <16 x i8> [[BC1]], ; AVX-NEXT: [[BC2:%.*]] = bitcast <16 x i8> [[ADD]] to <4 x i32> @@ -164,8 +163,7 @@ define <2 x i64> @PR35454_2(<2 x i64> %v) { ; SSE-NEXT: ret <2 x i64> [[BC3]] ; ; AVX-LABEL: @PR35454_2( -; AVX-NEXT: [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32> -; AVX-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[BC]] to <8 x i16> +; AVX-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[V:%.*]] to <8 x i16> ; AVX-NEXT: [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> ; AVX-NEXT: [[ADD:%.*]] = shl <8 x i16> [[BC1]], ; AVX-NEXT: [[BC2:%.*]] = bitcast <8 x i16> [[ADD]] to <4 x i32> -- GitLab From 5b66b6a32ad89562732ad6a81c84783486b6187a Mon Sep 17 00:00:00 2001 From: Ivan Butygin Date: Tue, 2 Apr 2024 13:30:45 +0300 Subject: [PATCH 0178/1690] [mlir][pass] Add composite pass utility (#87166) Composite pass allows to run sequence of passes in the loop until fixed point or maximum number of iterations is reached. The usual candidates are canonicalize+CSE as canonicalize can open more opportunities for CSE and vice-versa. --- mlir/include/mlir/Transforms/Passes.h | 7 ++ mlir/include/mlir/Transforms/Passes.td | 17 +++ mlir/lib/Transforms/CMakeLists.txt | 1 + mlir/lib/Transforms/CompositePass.cpp | 105 ++++++++++++++++++ mlir/test/Transforms/composite-pass.mlir | 26 +++++ mlir/test/lib/Transforms/CMakeLists.txt | 1 + .../test/lib/Transforms/TestCompositePass.cpp | 38 +++++++ mlir/tools/mlir-opt/mlir-opt.cpp | 2 + 8 files changed, 197 insertions(+) create mode 100644 mlir/lib/Transforms/CompositePass.cpp create mode 100644 mlir/test/Transforms/composite-pass.mlir create mode 100644 mlir/test/lib/Transforms/TestCompositePass.cpp diff --git a/mlir/include/mlir/Transforms/Passes.h b/mlir/include/mlir/Transforms/Passes.h index 11f5b23e62c6..58bd61b2ae8b 100644 --- a/mlir/include/mlir/Transforms/Passes.h +++ b/mlir/include/mlir/Transforms/Passes.h @@ -43,6 +43,7 @@ class GreedyRewriteConfig; #define GEN_PASS_DECL_SYMBOLDCE #define GEN_PASS_DECL_SYMBOLPRIVATIZE #define GEN_PASS_DECL_TOPOLOGICALSORT +#define GEN_PASS_DECL_COMPOSITEFIXEDPOINTPASS #include "mlir/Transforms/Passes.h.inc" /// Creates an instance of the Canonicalizer pass, configured with default @@ -130,6 +131,12 @@ createSymbolPrivatizePass(ArrayRef excludeSymbols = {}); /// their producers. std::unique_ptr createTopologicalSortPass(); +/// Create composite pass, which runs provided set of passes until fixed point +/// or maximum number of iterations reached. +std::unique_ptr createCompositeFixedPointPass( + std::string name, llvm::function_ref populateFunc, + int maxIterations = 10); + //===----------------------------------------------------------------------===// // Registration //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td index 51b2a27da639..1b40a87c63f2 100644 --- a/mlir/include/mlir/Transforms/Passes.td +++ b/mlir/include/mlir/Transforms/Passes.td @@ -552,4 +552,21 @@ def TopologicalSort : Pass<"topological-sort"> { let constructor = "mlir::createTopologicalSortPass()"; } +def CompositeFixedPointPass : Pass<"composite-fixed-point-pass"> { + let summary = "Composite fixed point pass"; + let description = [{ + Composite pass runs provided set of passes until fixed point or maximum + number of iterations reached. + }]; + + let options = [ + Option<"name", "name", "std::string", /*default=*/"\"CompositeFixedPointPass\"", + "Composite pass display name">, + Option<"pipelineStr", "pipeline", "std::string", /*default=*/"", + "Composite pass inner pipeline">, + Option<"maxIter", "max-iterations", "int", /*default=*/"10", + "Maximum number of iterations if inner pipeline">, + ]; +} + #endif // MLIR_TRANSFORMS_PASSES diff --git a/mlir/lib/Transforms/CMakeLists.txt b/mlir/lib/Transforms/CMakeLists.txt index 6c32ecf8a2a2..90c0298fb5e4 100644 --- a/mlir/lib/Transforms/CMakeLists.txt +++ b/mlir/lib/Transforms/CMakeLists.txt @@ -2,6 +2,7 @@ add_subdirectory(Utils) add_mlir_library(MLIRTransforms Canonicalizer.cpp + CompositePass.cpp ControlFlowSink.cpp CSE.cpp GenerateRuntimeVerification.cpp diff --git a/mlir/lib/Transforms/CompositePass.cpp b/mlir/lib/Transforms/CompositePass.cpp new file mode 100644 index 000000000000..b388a28da642 --- /dev/null +++ b/mlir/lib/Transforms/CompositePass.cpp @@ -0,0 +1,105 @@ +//===- CompositePass.cpp - Composite pass code ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// CompositePass allows to run set of passes until fixed point is reached. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Transforms/Passes.h" + +#include "mlir/Pass/Pass.h" +#include "mlir/Pass/PassManager.h" + +namespace mlir { +#define GEN_PASS_DEF_COMPOSITEFIXEDPOINTPASS +#include "mlir/Transforms/Passes.h.inc" +} // namespace mlir + +using namespace mlir; + +namespace { +struct CompositeFixedPointPass final + : public impl::CompositeFixedPointPassBase { + using CompositeFixedPointPassBase::CompositeFixedPointPassBase; + + CompositeFixedPointPass( + std::string name_, llvm::function_ref populateFunc, + int maxIterations) { + name = std::move(name_); + maxIter = maxIterations; + populateFunc(dynamicPM); + + llvm::raw_string_ostream os(pipelineStr); + dynamicPM.printAsTextualPipeline(os); + } + + LogicalResult initializeOptions( + StringRef options, + function_ref errorHandler) override { + if (failed(CompositeFixedPointPassBase::initializeOptions(options, + errorHandler))) + return failure(); + + if (failed(parsePassPipeline(pipelineStr, dynamicPM))) + return errorHandler("Failed to parse composite pass pipeline"); + + return success(); + } + + LogicalResult initialize(MLIRContext *context) override { + if (maxIter <= 0) + return emitError(UnknownLoc::get(context)) + << "Invalid maxIterations value: " << maxIter << "\n"; + + return success(); + } + + void getDependentDialects(DialectRegistry ®istry) const override { + dynamicPM.getDependentDialects(registry); + } + + void runOnOperation() override { + auto op = getOperation(); + OperationFingerPrint fp(op); + + int currentIter = 0; + int maxIterVal = maxIter; + while (true) { + if (failed(runPipeline(dynamicPM, op))) + return signalPassFailure(); + + if (currentIter++ >= maxIterVal) { + op->emitWarning("Composite pass \"" + llvm::Twine(name) + + "\"+ didn't converge in " + llvm::Twine(maxIterVal) + + " iterations"); + break; + } + + OperationFingerPrint newFp(op); + if (newFp == fp) + break; + + fp = newFp; + } + } + +protected: + llvm::StringRef getName() const override { return name; } + +private: + OpPassManager dynamicPM; +}; +} // namespace + +std::unique_ptr mlir::createCompositeFixedPointPass( + std::string name, llvm::function_ref populateFunc, + int maxIterations) { + + return std::make_unique(std::move(name), + populateFunc, maxIterations); +} diff --git a/mlir/test/Transforms/composite-pass.mlir b/mlir/test/Transforms/composite-pass.mlir new file mode 100644 index 000000000000..829470c2c9aa --- /dev/null +++ b/mlir/test/Transforms/composite-pass.mlir @@ -0,0 +1,26 @@ +// RUN: mlir-opt %s --log-actions-to=- --test-composite-fixed-point-pass -split-input-file | FileCheck %s +// RUN: mlir-opt %s --log-actions-to=- --composite-fixed-point-pass='name=TestCompositePass pipeline=any(canonicalize,cse)' -split-input-file | FileCheck %s + +// CHECK-LABEL: running `TestCompositePass` +// CHECK: running `Canonicalizer` +// CHECK: running `CSE` +// CHECK-NOT: running `Canonicalizer` +// CHECK-NOT: running `CSE` +func.func @test() { + return +} + +// ----- + +// CHECK-LABEL: running `TestCompositePass` +// CHECK: running `Canonicalizer` +// CHECK: running `CSE` +// CHECK: running `Canonicalizer` +// CHECK: running `CSE` +// CHECK-NOT: running `Canonicalizer` +// CHECK-NOT: running `CSE` +func.func @test() { +// this constant will be canonicalized away, causing another pass iteration + %0 = arith.constant 1.5 : f32 + return +} diff --git a/mlir/test/lib/Transforms/CMakeLists.txt b/mlir/test/lib/Transforms/CMakeLists.txt index 2a3a8608db54..a849b7ebd29e 100644 --- a/mlir/test/lib/Transforms/CMakeLists.txt +++ b/mlir/test/lib/Transforms/CMakeLists.txt @@ -20,6 +20,7 @@ endif() # Exclude tests from libMLIR.so add_mlir_library(MLIRTestTransforms TestCommutativityUtils.cpp + TestCompositePass.cpp TestConstantFold.cpp TestControlFlowSink.cpp TestInlining.cpp diff --git a/mlir/test/lib/Transforms/TestCompositePass.cpp b/mlir/test/lib/Transforms/TestCompositePass.cpp new file mode 100644 index 000000000000..5c0d93cc0d64 --- /dev/null +++ b/mlir/test/lib/Transforms/TestCompositePass.cpp @@ -0,0 +1,38 @@ +//===------ TestCompositePass.cpp --- composite test pass -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a pass to test the composite pass utility. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Pass/Pass.h" +#include "mlir/Pass/PassManager.h" +#include "mlir/Pass/PassRegistry.h" +#include "mlir/Transforms/Passes.h" + +namespace mlir { +namespace test { +void registerTestCompositePass() { + registerPassPipeline( + "test-composite-fixed-point-pass", "Test composite pass", + [](OpPassManager &pm, StringRef optionsStr, + function_ref errorHandler) { + if (!optionsStr.empty()) + return failure(); + + pm.addPass(createCompositeFixedPointPass( + "TestCompositePass", [](OpPassManager &p) { + p.addPass(createCanonicalizerPass()); + p.addPass(createCSEPass()); + })); + return success(); + }, + [](function_ref) {}); +} +} // namespace test +} // namespace mlir diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp index 82b3881792bf..6ce9f3041d6f 100644 --- a/mlir/tools/mlir-opt/mlir-opt.cpp +++ b/mlir/tools/mlir-opt/mlir-opt.cpp @@ -68,6 +68,7 @@ void registerTosaTestQuantUtilAPIPass(); void registerVectorizerTestPass(); namespace test { +void registerTestCompositePass(); void registerCommutativityUtils(); void registerConvertCallOpPass(); void registerInliner(); @@ -195,6 +196,7 @@ void registerTestPasses() { registerVectorizerTestPass(); registerTosaTestQuantUtilAPIPass(); + mlir::test::registerTestCompositePass(); mlir::test::registerCommutativityUtils(); mlir::test::registerConvertCallOpPass(); mlir::test::registerInliner(); -- GitLab From 2f48a1ff574573e7be170d39ab8de79d9db8bcea Mon Sep 17 00:00:00 2001 From: David Spickett Date: Tue, 2 Apr 2024 12:13:59 +0100 Subject: [PATCH 0179/1690] [lldb][FreeBSD] Add FreeBSD specific AT_HWCAP value (#84147) While adding register fields I realised that the AUXV values for Linux and FreeBSD disagree here. So I've added a FreeBSD specific HWCAP value that I can use from FreeBSD specific code. The alternative is translating GetAuxValue calls depending on platform, which requires that we know what we are at all times. Another way would be to convert the entries' values when we construct the AuxVector but the platform specific call that reads the data just returns a raw array. So adding another layer here is more disruption. --- lldb/source/Plugins/Process/Utility/AuxVector.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lldb/source/Plugins/Process/Utility/AuxVector.h b/lldb/source/Plugins/Process/Utility/AuxVector.h index 3b0f55d35e5d..4175cb73b234 100644 --- a/lldb/source/Plugins/Process/Utility/AuxVector.h +++ b/lldb/source/Plugins/Process/Utility/AuxVector.h @@ -20,9 +20,9 @@ public: AuxVector(const lldb_private::DataExtractor &data); /// Constants describing the type of entry. - /// On Linux, running "LD_SHOW_AUXV=1 ./executable" will spew AUX + /// On Linux and FreeBSD, running "LD_SHOW_AUXV=1 ./executable" will spew AUX /// information. Added AUXV prefix to avoid potential conflicts with system- - /// defined macros + /// defined macros. For FreeBSD, the numbers can be found in sys/elf_common.h. enum EntryType { AUXV_AT_NULL = 0, ///< End of auxv. AUXV_AT_IGNORE = 1, ///< Ignore entry. @@ -39,6 +39,11 @@ public: AUXV_AT_EUID = 12, ///< Effective UID. AUXV_AT_GID = 13, ///< GID. AUXV_AT_EGID = 14, ///< Effective GID. + + // At this point Linux and FreeBSD diverge and many of the following values + // are Linux specific. If you use them make sure you are in Linux specific + // code or they have the same value on other platforms. + AUXV_AT_CLKTCK = 17, ///< Clock frequency (e.g. times(2)). AUXV_AT_PLATFORM = 15, ///< String identifying platform. AUXV_AT_HWCAP = @@ -60,6 +65,10 @@ public: AUXV_AT_L1D_CACHESHAPE = 35, AUXV_AT_L2_CACHESHAPE = 36, AUXV_AT_L3_CACHESHAPE = 37, + + // Platform specific values which may overlap the Linux values. + + AUXV_FREEBSD_AT_HWCAP = 25, ///< FreeBSD specific AT_HWCAP value. }; std::optional GetAuxValue(enum EntryType entry_type) const; -- GitLab From 198c3eecee50d90cdff4b7840cfa39eef5613870 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Tue, 2 Apr 2024 13:19:51 +0200 Subject: [PATCH 0180/1690] [bazel] Fix the format of libc_build_rules.bzl --- utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl index 7dc12bade260..80cf59d7ef12 100644 --- a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl +++ b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl @@ -92,7 +92,7 @@ def libc_function( # x86 targets have -mno-omit-leaf-frame-pointer. platform_copts = selects.with_or({ PLATFORM_CPU_X86_64: ["-mno-omit-leaf-frame-pointer"], - "//conditions:default": [] + "//conditions:default": [], }) copts = copts + platform_copts -- GitLab From a88a4da61a8eb3378bc333602d5b7e56a24cfb66 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Tue, 2 Apr 2024 12:21:57 +0100 Subject: [PATCH 0181/1690] [lldb] clang-format AuxVector.h (#85057) Doing this in its own commit so the intent of 2f48a1ff574573e7be170d39ab8de79d9db8bcea is clearer. --- .../Plugins/Process/Utility/AuxVector.h | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lldb/source/Plugins/Process/Utility/AuxVector.h b/lldb/source/Plugins/Process/Utility/AuxVector.h index 4175cb73b234..2670b34f6b0a 100644 --- a/lldb/source/Plugins/Process/Utility/AuxVector.h +++ b/lldb/source/Plugins/Process/Utility/AuxVector.h @@ -24,21 +24,21 @@ public: /// information. Added AUXV prefix to avoid potential conflicts with system- /// defined macros. For FreeBSD, the numbers can be found in sys/elf_common.h. enum EntryType { - AUXV_AT_NULL = 0, ///< End of auxv. - AUXV_AT_IGNORE = 1, ///< Ignore entry. - AUXV_AT_EXECFD = 2, ///< File descriptor of program. - AUXV_AT_PHDR = 3, ///< Program headers. - AUXV_AT_PHENT = 4, ///< Size of program header. - AUXV_AT_PHNUM = 5, ///< Number of program headers. - AUXV_AT_PAGESZ = 6, ///< Page size. - AUXV_AT_BASE = 7, ///< Interpreter base address. - AUXV_AT_FLAGS = 8, ///< Flags. - AUXV_AT_ENTRY = 9, ///< Program entry point. - AUXV_AT_NOTELF = 10, ///< Set if program is not an ELF. - AUXV_AT_UID = 11, ///< UID. - AUXV_AT_EUID = 12, ///< Effective UID. - AUXV_AT_GID = 13, ///< GID. - AUXV_AT_EGID = 14, ///< Effective GID. + AUXV_AT_NULL = 0, ///< End of auxv. + AUXV_AT_IGNORE = 1, ///< Ignore entry. + AUXV_AT_EXECFD = 2, ///< File descriptor of program. + AUXV_AT_PHDR = 3, ///< Program headers. + AUXV_AT_PHENT = 4, ///< Size of program header. + AUXV_AT_PHNUM = 5, ///< Number of program headers. + AUXV_AT_PAGESZ = 6, ///< Page size. + AUXV_AT_BASE = 7, ///< Interpreter base address. + AUXV_AT_FLAGS = 8, ///< Flags. + AUXV_AT_ENTRY = 9, ///< Program entry point. + AUXV_AT_NOTELF = 10, ///< Set if program is not an ELF. + AUXV_AT_UID = 11, ///< UID. + AUXV_AT_EUID = 12, ///< Effective UID. + AUXV_AT_GID = 13, ///< GID. + AUXV_AT_EGID = 14, ///< Effective GID. // At this point Linux and FreeBSD diverge and many of the following values // are Linux specific. If you use them make sure you are in Linux specific -- GitLab From 9a05a89d1ef73de7ab787071931f449935d841a7 Mon Sep 17 00:00:00 2001 From: Carlos Alberto Enciso Date: Tue, 2 Apr 2024 12:34:31 +0100 Subject: [PATCH 0182/1690] [speculative-execution] Hoists debug values unnecessarily. (#85782) After https://reviews.llvm.org/D81730: `SpeculativeExecutionPass::considerHoistingFromTo` hoists instructions, including debug intrinsics, as long as none of their used values are instructions that appear prior in the block that are not being hoisted. This behaviour has been duplicated for DPValues to get rid of a binary difference. The correct solution is not hoist these debug values at all, whichever format they're in. --- .../Scalar/SpeculativeExecution.cpp | 51 +++++++++---------- .../SpeculativeExecution/PR46267.ll | 2 +- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp b/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp index 5efc340da60b..f921ee72a0a1 100644 --- a/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp +++ b/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp @@ -260,11 +260,31 @@ static InstructionCost ComputeSpeculationCost(const Instruction *I, } } +// Do not hoist any debug info intrinsics. +// ... +// if (cond) { +// x = y * z; +// foo(); +// } +// ... +// -------- Which then becomes: +// ... +// if.then: +// %x = mul i32 %y, %z +// call void @llvm.dbg.value(%x, !"x", !DIExpression()) +// call void foo() +// +// SpeculativeExecution might decide to hoist the 'y * z' calculation +// out of the 'if' block, because it is more efficient that way, so the +// '%x = mul i32 %y, %z' moves to the block above. But it might also +// decide to hoist the 'llvm.dbg.value' call. +// This is incorrect, because even if we've moved the calculation of +// 'y * z', we should not see the value of 'x' change unless we +// actually go inside the 'if' block. + bool SpeculativeExecutionPass::considerHoistingFromTo( BasicBlock &FromBlock, BasicBlock &ToBlock) { SmallPtrSet NotHoisted; - SmallDenseMap> - DbgVariableRecordsToHoist; auto HasNoUnhoistedInstr = [&NotHoisted](auto Values) { for (const Value *V : Values) { if (const auto *I = dyn_cast_or_null(V)) @@ -275,15 +295,8 @@ bool SpeculativeExecutionPass::considerHoistingFromTo( }; auto AllPrecedingUsesFromBlockHoisted = [&HasNoUnhoistedInstr](const User *U) { - // Debug variable has special operand to check it's not hoisted. - if (const auto *DVI = dyn_cast(U)) - return HasNoUnhoistedInstr(DVI->location_ops()); - - // Usially debug label intrinsic corresponds to label in LLVM IR. In - // these cases we should not move it here. - // TODO: Possible special processing needed to detect it is related to a - // hoisted instruction. - if (isa(U)) + // Do not hoist any debug info intrinsics. + if (isa(U)) return false; return HasNoUnhoistedInstr(U->operand_values()); @@ -292,12 +305,6 @@ bool SpeculativeExecutionPass::considerHoistingFromTo( InstructionCost TotalSpeculationCost = 0; unsigned NotHoistedInstCount = 0; for (const auto &I : FromBlock) { - // Make note of any DbgVariableRecords that need hoisting. DbgLabelRecords - // get left behind just like llvm.dbg.labels. - for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) { - if (HasNoUnhoistedInstr(DVR.location_ops())) - DbgVariableRecordsToHoist[DVR.getInstruction()].push_back(&DVR); - } const InstructionCost Cost = ComputeSpeculationCost(&I, *TTI); if (Cost.isValid() && isSafeToSpeculativelyExecute(&I) && AllPrecedingUsesFromBlockHoisted(&I)) { @@ -315,16 +322,6 @@ bool SpeculativeExecutionPass::considerHoistingFromTo( } for (auto I = FromBlock.begin(); I != FromBlock.end();) { - // If any DbgVariableRecords attached to this instruction should be hoisted, - // hoist them now - they will end up attached to either the next hoisted - // instruction or the ToBlock terminator. - if (DbgVariableRecordsToHoist.contains(&*I)) { - for (auto *DVR : DbgVariableRecordsToHoist[&*I]) { - DVR->removeFromParent(); - ToBlock.insertDbgRecordBefore(DVR, - ToBlock.getTerminator()->getIterator()); - } - } // We have to increment I before moving Current as moving Current // changes the list that I is iterating through. auto Current = I; diff --git a/llvm/test/Transforms/SpeculativeExecution/PR46267.ll b/llvm/test/Transforms/SpeculativeExecution/PR46267.ll index d940ee6a7863..69dac2220d9a 100644 --- a/llvm/test/Transforms/SpeculativeExecution/PR46267.ll +++ b/llvm/test/Transforms/SpeculativeExecution/PR46267.ll @@ -31,7 +31,6 @@ define void @f(i32 %i) { entry: ; CHECK-LABEL: @f( ; CHECK: %a2 = add i32 %i, 0 -; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 %a2 br i1 undef, label %land.rhs, label %land.end land.rhs: ; preds = %entry @@ -42,6 +41,7 @@ land.rhs: ; preds = %entry ; CHECK-NEXT: %a0 = load i32, ptr undef, align 1 ; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 %a0 ; CHECK-NEXT: call void @llvm.dbg.label +; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 %a2 call void @llvm.dbg.label(metadata !11), !dbg !10 %y = alloca i32, align 4 call void @llvm.dbg.declare(metadata ptr %y, metadata !14, metadata !DIExpression()), !dbg !10 -- GitLab From 0b13e2c82315eac8926f1c4497c4d56a507c3999 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Tue, 2 Apr 2024 13:44:38 +0200 Subject: [PATCH 0183/1690] [bazel] Another format fix for libc_build_rules.bzl, NFC --- utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl index 80cf59d7ef12..be59e18ffd89 100644 --- a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl +++ b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl @@ -78,6 +78,7 @@ def libc_function( its deps. **kwargs: Other attributes relevant for a cc_library. For example, deps. """ + # We use the explicit equals pattern here because append and += mutate the # original list, where this creates a new list and stores it in deps. copts = copts or [] @@ -89,6 +90,7 @@ def libc_function( "-fno-omit-frame-pointer", "-fstack-protector-strong", ] + # x86 targets have -mno-omit-leaf-frame-pointer. platform_copts = selects.with_or({ PLATFORM_CPU_X86_64: ["-mno-omit-leaf-frame-pointer"], -- GitLab From 7ef602b58c1ccacab20d9d01e24b281458c3facc Mon Sep 17 00:00:00 2001 From: Sam McCall Date: Tue, 2 Apr 2024 13:48:45 +0200 Subject: [PATCH 0184/1690] Reapply "[clang][nullability] allow _Nonnull etc on nullable class types (#82705)" (#87325) This reverts commit 28760b63bbf9e267713957105a8d17091fb0d20e. The last commit was missing the new testcase, now fixed. --- clang/docs/ReleaseNotes.rst | 15 +++++ clang/include/clang/Basic/Attr.td | 3 +- clang/include/clang/Basic/AttrDocs.td | 25 ++++++++ clang/include/clang/Basic/Features.def | 1 + clang/include/clang/Parse/Parser.h | 1 + clang/include/clang/Sema/Sema.h | 3 + clang/lib/AST/Type.cpp | 29 ++++++--- clang/lib/CodeGen/CGCall.cpp | 3 +- clang/lib/CodeGen/CodeGenFunction.cpp | 3 +- clang/lib/Parse/ParseDeclCXX.cpp | 33 +++++++--- clang/lib/Sema/SemaAttr.cpp | 12 ++++ clang/lib/Sema/SemaChecking.cpp | 9 +++ clang/lib/Sema/SemaDecl.cpp | 4 +- clang/lib/Sema/SemaDeclAttr.cpp | 18 ++++++ clang/lib/Sema/SemaInit.cpp | 5 ++ clang/lib/Sema/SemaOverload.cpp | 7 +++ clang/lib/Sema/SemaTemplate.cpp | 1 + clang/lib/Sema/SemaType.cpp | 18 ++++-- clang/test/Sema/nullability.c | 2 + clang/test/SemaCXX/nullability.cpp | 62 ++++++++++++++++++- .../Inputs/nullability-consistency-smart.h | 7 +++ .../SemaObjCXX/nullability-consistency.mm | 1 + 22 files changed, 233 insertions(+), 29 deletions(-) create mode 100644 clang/test/SemaObjCXX/Inputs/nullability-consistency-smart.h diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 76eaf0bf11c3..b2faab1f1525 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -253,6 +253,21 @@ Attribute Changes in Clang added a new extension query ``__has_extension(swiftcc)`` corresponding to the ``__attribute__((swiftcc))`` attribute. +- The ``_Nullable`` and ``_Nonnull`` family of type attributes can now apply + to certain C++ class types, such as smart pointers: + ``void useObject(std::unique_ptr _Nonnull obj);``. + + This works for standard library types including ``unique_ptr``, ``shared_ptr``, + and ``function``. See + `the attribute reference documentation `_ + for the full list. + +- The ``_Nullable`` attribute can be applied to C++ class declarations: + ``template class _Nullable MySmartPointer {};``. + + This allows the ``_Nullable`` and ``_Nonnull`` family of type attributes to + apply to this class. + Improvements to Clang's diagnostics ----------------------------------- - Clang now applies syntax highlighting to the code snippets it diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index 80e607525a0a..6584460cf568 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -2178,9 +2178,10 @@ def TypeNonNull : TypeAttr { let Documentation = [TypeNonNullDocs]; } -def TypeNullable : TypeAttr { +def TypeNullable : DeclOrTypeAttr { let Spellings = [CustomKeyword<"_Nullable">]; let Documentation = [TypeNullableDocs]; +// let Subjects = SubjectList<[CXXRecord], ErrorDiag>; } def TypeNullableResult : TypeAttr { diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td index 3ea4d676b4f8..0ca4ea377fc3 100644 --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -4151,6 +4151,20 @@ non-underscored keywords. For example: @property (assign, nullable) NSView *superview; @property (readonly, nonnull) NSArray *subviews; @end + +As well as built-in pointer types, the nullability attributes can be attached +to C++ classes marked with the ``_Nullable`` attribute. + +The following C++ standard library types are considered nullable: +``unique_ptr``, ``shared_ptr``, ``auto_ptr``, ``exception_ptr``, ``function``, +``move_only_function`` and ``coroutine_handle``. + +Types should be marked nullable only where the type itself leaves nullability +ambiguous. For example, ``std::optional`` is not marked ``_Nullable``, because +``optional _Nullable`` is redundant and ``optional _Nonnull`` is +not a useful type. ``std::weak_ptr`` is not nullable, because its nullability +can change with no visible modification, so static annotation is unlikely to be +unhelpful. }]; } @@ -4185,6 +4199,17 @@ The ``_Nullable`` nullability qualifier indicates that a value of the int fetch_or_zero(int * _Nullable ptr); a caller of ``fetch_or_zero`` can provide null. + +The ``_Nullable`` attribute on classes indicates that the given class can +represent null values, and so the ``_Nullable``, ``_Nonnull`` etc qualifiers +make sense for this type. For example: + + .. code-block:: c + + class _Nullable ArenaPointer { ... }; + + ArenaPointer _Nonnull x = ...; + ArenaPointer _Nullable y = nullptr; }]; } diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def index b41aadc73f20..fe4d1c4afcca 100644 --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -94,6 +94,7 @@ EXTENSION(define_target_os_macros, FEATURE(enumerator_attributes, true) FEATURE(nullability, true) FEATURE(nullability_on_arrays, true) +FEATURE(nullability_on_classes, true) FEATURE(nullability_nullable_result, true) FEATURE(memory_sanitizer, LangOpts.Sanitize.hasOneOf(SanitizerKind::Memory | diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h index bba8ef4ff017..580bf2a5d79d 100644 --- a/clang/include/clang/Parse/Parser.h +++ b/clang/include/clang/Parse/Parser.h @@ -3014,6 +3014,7 @@ private: void DiagnoseAndSkipExtendedMicrosoftTypeAttributes(); SourceLocation SkipExtendedMicrosoftTypeAttributes(); void ParseMicrosoftInheritanceClassAttributes(ParsedAttributes &attrs); + void ParseNullabilityClassAttributes(ParsedAttributes &attrs); void ParseBorlandTypeAttributes(ParsedAttributes &attrs); void ParseOpenCLKernelAttributes(ParsedAttributes &attrs); void ParseOpenCLQualifiers(ParsedAttributes &Attrs); diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index a02b684f2c77..8c98d8c7fef7 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -1660,6 +1660,9 @@ public: /// Add [[gsl::Pointer]] attributes for std:: types. void inferGslPointerAttribute(TypedefNameDecl *TD); + /// Add _Nullable attributes for std:: types. + void inferNullableClassAttribute(CXXRecordDecl *CRD); + enum PragmaOptionsAlignKind { POAK_Native, // #pragma options align=native POAK_Natural, // #pragma options align=natural diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp index 779d8a810820..cb22c91a12aa 100644 --- a/clang/lib/AST/Type.cpp +++ b/clang/lib/AST/Type.cpp @@ -4652,16 +4652,15 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const { case Type::Auto: return ResultIfUnknown; - // Dependent template specializations can instantiate to pointer - // types unless they're known to be specializations of a class - // template. + // Dependent template specializations could instantiate to pointer types. case Type::TemplateSpecialization: - if (TemplateDecl *templateDecl - = cast(type.getTypePtr()) - ->getTemplateName().getAsTemplateDecl()) { - if (isa(templateDecl)) - return false; - } + // If it's a known class template, we can already check if it's nullable. + if (TemplateDecl *templateDecl = + cast(type.getTypePtr()) + ->getTemplateName() + .getAsTemplateDecl()) + if (auto *CTD = dyn_cast(templateDecl)) + return CTD->getTemplatedDecl()->hasAttr(); return ResultIfUnknown; case Type::Builtin: @@ -4718,6 +4717,17 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const { } llvm_unreachable("unknown builtin type"); + case Type::Record: { + const RecordDecl *RD = cast(type)->getDecl(); + // For template specializations, look only at primary template attributes. + // This is a consistent regardless of whether the instantiation is known. + if (const auto *CTSD = dyn_cast(RD)) + return CTSD->getSpecializedTemplate() + ->getTemplatedDecl() + ->hasAttr(); + return RD->hasAttr(); + } + // Non-pointer types. case Type::Complex: case Type::LValueReference: @@ -4735,7 +4745,6 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const { case Type::DependentAddressSpace: case Type::FunctionProto: case Type::FunctionNoProto: - case Type::Record: case Type::DeducedTemplateSpecialization: case Type::Enum: case Type::InjectedClassName: diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index 9308528ac938..f12765b82693 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -4379,7 +4379,8 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType, NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo); bool CanCheckNullability = false; - if (SanOpts.has(SanitizerKind::NullabilityArg) && !NNAttr && PVD) { + if (SanOpts.has(SanitizerKind::NullabilityArg) && !NNAttr && PVD && + !PVD->getType()->isRecordType()) { auto Nullability = PVD->getType()->getNullability(); CanCheckNullability = Nullability && *Nullability == NullabilityKind::NonNull && diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index 90324de7268e..6474d6c8c1d1 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -990,7 +990,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, // return value. Initialize the flag to 'true' and refine it in EmitParmDecl. if (SanOpts.has(SanitizerKind::NullabilityReturn)) { auto Nullability = FnRetTy->getNullability(); - if (Nullability && *Nullability == NullabilityKind::NonNull) { + if (Nullability && *Nullability == NullabilityKind::NonNull && + !FnRetTy->isRecordType()) { if (!(SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl && CurCodeDecl->getAttr())) RetValNullabilityPrecondition = diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp index 63fe678cbb29..861a25dc5103 100644 --- a/clang/lib/Parse/ParseDeclCXX.cpp +++ b/clang/lib/Parse/ParseDeclCXX.cpp @@ -1502,6 +1502,15 @@ void Parser::ParseMicrosoftInheritanceClassAttributes(ParsedAttributes &attrs) { } } +void Parser::ParseNullabilityClassAttributes(ParsedAttributes &attrs) { + while (Tok.is(tok::kw__Nullable)) { + IdentifierInfo *AttrName = Tok.getIdentifierInfo(); + auto Kind = Tok.getKind(); + SourceLocation AttrNameLoc = ConsumeToken(); + attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0, Kind); + } +} + /// Determine whether the following tokens are valid after a type-specifier /// which could be a standalone declaration. This will conservatively return /// true if there's any doubt, and is appropriate for insert-';' fixits. @@ -1683,15 +1692,21 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind, ParsedAttributes attrs(AttrFactory); // If attributes exist after tag, parse them. - MaybeParseAttributes(PAKM_CXX11 | PAKM_Declspec | PAKM_GNU, attrs); - - // Parse inheritance specifiers. - if (Tok.isOneOf(tok::kw___single_inheritance, tok::kw___multiple_inheritance, - tok::kw___virtual_inheritance)) - ParseMicrosoftInheritanceClassAttributes(attrs); - - // Allow attributes to precede or succeed the inheritance specifiers. - MaybeParseAttributes(PAKM_CXX11 | PAKM_Declspec | PAKM_GNU, attrs); + for (;;) { + MaybeParseAttributes(PAKM_CXX11 | PAKM_Declspec | PAKM_GNU, attrs); + // Parse inheritance specifiers. + if (Tok.isOneOf(tok::kw___single_inheritance, + tok::kw___multiple_inheritance, + tok::kw___virtual_inheritance)) { + ParseMicrosoftInheritanceClassAttributes(attrs); + continue; + } + if (Tok.is(tok::kw__Nullable)) { + ParseNullabilityClassAttributes(attrs); + continue; + } + break; + } // Source location used by FIXIT to insert misplaced // C++11 attributes diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp index 0dcf42e48997..a5dd158808f2 100644 --- a/clang/lib/Sema/SemaAttr.cpp +++ b/clang/lib/Sema/SemaAttr.cpp @@ -215,6 +215,18 @@ void Sema::inferGslOwnerPointerAttribute(CXXRecordDecl *Record) { inferGslPointerAttribute(Record, Record); } +void Sema::inferNullableClassAttribute(CXXRecordDecl *CRD) { + static llvm::StringSet<> Nullable{ + "auto_ptr", "shared_ptr", "unique_ptr", "exception_ptr", + "coroutine_handle", "function", "move_only_function", + }; + + if (CRD->isInStdNamespace() && Nullable.count(CRD->getName()) && + !CRD->hasAttr()) + for (Decl *Redecl : CRD->redecls()) + Redecl->addAttr(TypeNullableAttr::CreateImplicit(Context)); +} + void Sema::ActOnPragmaOptionsAlign(PragmaOptionsAlignKind Kind, SourceLocation PragmaLoc) { PragmaMsStackAction Action = Sema::PSK_Reset; diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 11401b6f56c0..3dcd18b3afc8 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -27,6 +27,7 @@ #include "clang/AST/ExprObjC.h" #include "clang/AST/ExprOpenMP.h" #include "clang/AST/FormatString.h" +#include "clang/AST/IgnoreExpr.h" #include "clang/AST/NSAPI.h" #include "clang/AST/NonTrivialTypeVisitor.h" #include "clang/AST/OperationKinds.h" @@ -7610,6 +7611,14 @@ bool Sema::getFormatStringInfo(const FormatAttr *Format, bool IsCXXMember, /// /// Returns true if the value evaluates to null. static bool CheckNonNullExpr(Sema &S, const Expr *Expr) { + // Treat (smart) pointers constructed from nullptr as null, whether we can + // const-evaluate them or not. + // This must happen first: the smart pointer expr might have _Nonnull type! + if (isa( + IgnoreExprNodes(Expr, IgnoreImplicitAsWrittenSingleStep, + IgnoreElidableImplicitConstructorSingleStep))) + return true; + // If the expression has non-null type, it doesn't evaluate to null. if (auto nullability = Expr->IgnoreImplicit()->getType()->getNullability()) { if (*nullability == NullabilityKind::NonNull) diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 5027deda0d7e..6ff85c0c5c29 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -18319,8 +18319,10 @@ CreateNewDecl: if (PrevDecl) mergeDeclAttributes(New, PrevDecl); - if (auto *CXXRD = dyn_cast(New)) + if (auto *CXXRD = dyn_cast(New)) { inferGslOwnerPointerAttribute(CXXRD); + inferNullableClassAttribute(CXXRD); + } // If there's a #pragma GCC visibility in scope, set the visibility of this // record. diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index f25f3afd0f4a..8bce04640e74 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -5982,6 +5982,20 @@ static void handleBuiltinAliasAttr(Sema &S, Decl *D, D->addAttr(::new (S.Context) BuiltinAliasAttr(S.Context, AL, Ident)); } +static void handleNullableTypeAttr(Sema &S, Decl *D, const ParsedAttr &AL) { + if (AL.isUsedAsTypeAttr()) + return; + + if (auto *CRD = dyn_cast(D); + !CRD || !(CRD->isClass() || CRD->isStruct())) { + S.Diag(AL.getRange().getBegin(), diag::err_attribute_wrong_decl_type_str) + << AL << AL.isRegularKeywordAttribute() << "classes"; + return; + } + + handleSimpleAttribute(S, D, AL); +} + static void handlePreferredTypeAttr(Sema &S, Decl *D, const ParsedAttr &AL) { if (!AL.hasParsedType()) { S.Diag(AL.getLoc(), diag::err_attribute_wrong_number_arguments) << AL << 1; @@ -9933,6 +9947,10 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, case ParsedAttr::AT_UsingIfExists: handleSimpleAttribute(S, D, AL); break; + + case ParsedAttr::AT_TypeNullable: + handleNullableTypeAttr(S, D, AL); + break; } } diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index 777f89c70f87..e2a1951f1062 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -7082,6 +7082,11 @@ PerformConstructorInitialization(Sema &S, hasCopyOrMoveCtorParam(S.Context, getConstructorInfo(Step.Function.FoundDecl)); + // A smart pointer constructed from a nullable pointer is nullable. + if (NumArgs == 1 && !Kind.isExplicitCast()) + S.diagnoseNullableToNonnullConversion( + Entity.getType(), Args.front()->getType(), Kind.getLocation()); + // Determine the arguments required to actually perform the constructor // call. if (S.CompleteConstructorCall(Constructor, Step.Type, Args, Loc, diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp index 16d54c1ffe5f..0c913bc700f4 100644 --- a/clang/lib/Sema/SemaOverload.cpp +++ b/clang/lib/Sema/SemaOverload.cpp @@ -14826,6 +14826,13 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc, } } + // Check for nonnull = nullable. + // This won't be caught in the arg's initialization: the parameter to + // the assignment operator is not marked nonnull. + if (Op == OO_Equal) + diagnoseNullableToNonnullConversion(Args[0]->getType(), + Args[1]->getType(), OpLoc); + // Convert the arguments. if (CXXMethodDecl *Method = dyn_cast(FnDecl)) { // Best->Access is only meaningful for class members. diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index 1a2d5e9310db..befec401c8ee 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -2191,6 +2191,7 @@ DeclResult Sema::CheckClassTemplate( AddPushedVisibilityAttribute(NewClass); inferGslOwnerPointerAttribute(NewClass); + inferNullableClassAttribute(NewClass); if (TUK != TUK_Friend) { // Per C++ [basic.scope.temp]p2, skip the template parameter scopes. diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp index d88895d35294..8762744396f4 100644 --- a/clang/lib/Sema/SemaType.cpp +++ b/clang/lib/Sema/SemaType.cpp @@ -4717,6 +4717,18 @@ static bool DiagnoseMultipleAddrSpaceAttributes(Sema &S, LangAS ASOld, return false; } +// Whether this is a type broadly expected to have nullability attached. +// These types are affected by `#pragma assume_nonnull`, and missing nullability +// will be diagnosed with -Wnullability-completeness. +static bool shouldHaveNullability(QualType T) { + return T->canHaveNullability(/*ResultIfUnknown=*/false) && + // For now, do not infer/require nullability on C++ smart pointers. + // It's unclear whether the pragma's behavior is useful for C++. + // e.g. treating type-aliases and template-type-parameters differently + // from types of declarations can be surprising. + !isa(T->getCanonicalTypeInternal()); +} + static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state, QualType declSpecType, TypeSourceInfo *TInfo) { @@ -4835,8 +4847,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state, // inner pointers. complainAboutMissingNullability = CAMN_InnerPointers; - if (T->canHaveNullability(/*ResultIfUnknown*/ false) && - !T->getNullability()) { + if (shouldHaveNullability(T) && !T->getNullability()) { // Note that we allow but don't require nullability on dependent types. ++NumPointersRemaining; } @@ -5059,8 +5070,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state, // If the type itself could have nullability but does not, infer pointer // nullability and perform consistency checking. if (S.CodeSynthesisContexts.empty()) { - if (T->canHaveNullability(/*ResultIfUnknown*/ false) && - !T->getNullability()) { + if (shouldHaveNullability(T) && !T->getNullability()) { if (isVaList(T)) { // Record that we've seen a pointer, but do nothing else. if (NumPointersRemaining > 0) diff --git a/clang/test/Sema/nullability.c b/clang/test/Sema/nullability.c index 7d193bea4677..0401516233b6 100644 --- a/clang/test/Sema/nullability.c +++ b/clang/test/Sema/nullability.c @@ -248,3 +248,5 @@ void arraysInBlocks(void) { void (^withTypedefBad)(INTS _Nonnull [2]) = // expected-error {{nullability specifier '_Nonnull' cannot be applied to non-pointer type 'INTS' (aka 'int[4]')}} ^(INTS _Nonnull x[2]) {}; // expected-error {{nullability specifier '_Nonnull' cannot be applied to non-pointer type 'INTS' (aka 'int[4]')}} } + +struct _Nullable NotCplusplusClass {}; // expected-error {{'_Nullable' attribute only applies to classes}} diff --git a/clang/test/SemaCXX/nullability.cpp b/clang/test/SemaCXX/nullability.cpp index 8d0c4dc195a6..d52ba4efaccd 100644 --- a/clang/test/SemaCXX/nullability.cpp +++ b/clang/test/SemaCXX/nullability.cpp @@ -4,6 +4,10 @@ #else # error nullability feature should be defined #endif +#if __has_feature(nullability_on_classes) +#else +# error smart-pointer feature should be defined +#endif #include "nullability-completeness.h" @@ -27,6 +31,7 @@ template struct AddNonNull { typedef _Nonnull T type; // expected-error{{nullability specifier '_Nonnull' cannot be applied to non-pointer type 'int'}} // expected-error@-1{{nullability specifier '_Nonnull' cannot be applied to non-pointer type 'std::nullptr_t'}} + // expected-error@-2{{nullability specifier '_Nonnull' cannot be applied to non-pointer type 'NotPtr'}} }; typedef AddNonNull::type nonnull_int_ptr_1; @@ -35,6 +40,33 @@ typedef AddNonNull::type nonnull_int_ptr_3; // expected-note{{in inst typedef AddNonNull::type nonnull_non_pointer_1; // expected-note{{in instantiation of template class 'AddNonNull' requested here}} +// Nullability on C++ class types (smart pointers). +struct NotPtr{}; +typedef AddNonNull::type nonnull_non_pointer_2; // expected-note{{in instantiation}} +struct _Nullable SmartPtr{ + SmartPtr(); + SmartPtr(nullptr_t); + SmartPtr(const SmartPtr&); + SmartPtr(SmartPtr&&); + SmartPtr &operator=(const SmartPtr&); + SmartPtr &operator=(SmartPtr&&); +}; +typedef AddNonNull::type nonnull_smart_pointer_1; +template struct _Nullable SmartPtrTemplate{}; +typedef AddNonNull>::type nonnull_smart_pointer_2; +namespace std { inline namespace __1 { + template class unique_ptr {}; + template class function; + template class function {}; +} } +typedef AddNonNull>::type nonnull_smart_pointer_3; +typedef AddNonNull>::type nonnull_smart_pointer_4; + +class Derived : public SmartPtr {}; +Derived _Nullable x; // expected-error {{'_Nullable' cannot be applied}} +class DerivedPrivate : private SmartPtr {}; +DerivedPrivate _Nullable y; // expected-error {{'_Nullable' cannot be applied}} + // Non-null checking within a template. template struct AddNonNull2 { @@ -54,6 +86,7 @@ void (*& accepts_nonnull_2)(_Nonnull int *ptr) = accepts_nonnull_1; void (X::* accepts_nonnull_3)(_Nonnull int *ptr); void accepts_nonnull_4(_Nonnull int *ptr); void (&accepts_nonnull_5)(_Nonnull int *ptr) = accepts_nonnull_4; +void accepts_nonnull_6(SmartPtr _Nonnull); void test_accepts_nonnull_null_pointer_literal(X *x) { accepts_nonnull_1(0); // expected-warning{{null passed to a callee that requires a non-null argument}} @@ -61,6 +94,8 @@ void test_accepts_nonnull_null_pointer_literal(X *x) { (x->*accepts_nonnull_3)(0); // expected-warning{{null passed to a callee that requires a non-null argument}} accepts_nonnull_4(0); // expected-warning{{null passed to a callee that requires a non-null argument}} accepts_nonnull_5(0); // expected-warning{{null passed to a callee that requires a non-null argument}} + + accepts_nonnull_6(nullptr); // expected-warning{{null passed to a callee that requires a non-null argument}} } template @@ -71,6 +106,7 @@ void test_accepts_nonnull_null_pointer_literal_template() { template void test_accepts_nonnull_null_pointer_literal_template<&accepts_nonnull_4>(); // expected-note{{instantiation of function template specialization}} void TakeNonnull(void *_Nonnull); +void TakeSmartNonnull(SmartPtr _Nonnull); // Check different forms of assignment to a nonull type from a nullable one. void AssignAndInitNonNull() { void *_Nullable nullable; @@ -81,12 +117,26 @@ void AssignAndInitNonNull() { void *_Nonnull nonnull; nonnull = nullable; // expected-warning{{implicit conversion from nullable pointer 'void * _Nullable' to non-nullable pointer type 'void * _Nonnull'}} nonnull = {nullable}; // expected-warning{{implicit conversion from nullable pointer 'void * _Nullable' to non-nullable pointer type 'void * _Nonnull'}} - TakeNonnull(nullable); //expected-warning{{implicit conversion from nullable pointer 'void * _Nullable' to non-nullable pointer type 'void * _Nonnull}} TakeNonnull(nonnull); // OK + nonnull = (void *_Nonnull)nullable; // explicit cast OK + + SmartPtr _Nullable s_nullable; + SmartPtr _Nonnull s(s_nullable); // expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull'}} + SmartPtr _Nonnull s2{s_nullable}; // expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull'}} + SmartPtr _Nonnull s3 = {s_nullable}; // expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull'}} + SmartPtr _Nonnull s4 = s_nullable; // expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull'}} + SmartPtr _Nonnull s_nonnull; + s_nonnull = s_nullable; // expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull'}} + s_nonnull = {s_nullable}; // no warning here - might be nice? + TakeSmartNonnull(s_nullable); //expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull}} + TakeSmartNonnull(s_nonnull); // OK + s_nonnull = (SmartPtr _Nonnull)s_nullable; // explicit cast OK + s_nonnull = static_cast(s_nullable); // explicit cast OK } void *_Nullable ReturnNullable(); +SmartPtr _Nullable ReturnSmartNullable(); void AssignAndInitNonNullFromFn() { void *_Nonnull p(ReturnNullable()); // expected-warning{{implicit conversion from nullable pointer 'void * _Nullable' to non-nullable pointer type 'void * _Nonnull'}} @@ -96,8 +146,16 @@ void AssignAndInitNonNullFromFn() { void *_Nonnull nonnull; nonnull = ReturnNullable(); // expected-warning{{implicit conversion from nullable pointer 'void * _Nullable' to non-nullable pointer type 'void * _Nonnull'}} nonnull = {ReturnNullable()}; // expected-warning{{implicit conversion from nullable pointer 'void * _Nullable' to non-nullable pointer type 'void * _Nonnull'}} - TakeNonnull(ReturnNullable()); //expected-warning{{implicit conversion from nullable pointer 'void * _Nullable' to non-nullable pointer type 'void * _Nonnull}} + + SmartPtr _Nonnull s(ReturnSmartNullable()); // expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull'}} + SmartPtr _Nonnull s2{ReturnSmartNullable()}; // expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull'}} + SmartPtr _Nonnull s3 = {ReturnSmartNullable()}; // expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull'}} + SmartPtr _Nonnull s4 = ReturnSmartNullable(); // expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull'}} + SmartPtr _Nonnull s_nonnull; + s_nonnull = ReturnSmartNullable(); // expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull'}} + s_nonnull = {ReturnSmartNullable()}; + TakeSmartNonnull(ReturnSmartNullable()); // expected-warning{{implicit conversion from nullable pointer 'SmartPtr _Nullable' to non-nullable pointer type 'SmartPtr _Nonnull'}} } void ConditionalExpr(bool c) { diff --git a/clang/test/SemaObjCXX/Inputs/nullability-consistency-smart.h b/clang/test/SemaObjCXX/Inputs/nullability-consistency-smart.h new file mode 100644 index 000000000000..a28532e5d716 --- /dev/null +++ b/clang/test/SemaObjCXX/Inputs/nullability-consistency-smart.h @@ -0,0 +1,7 @@ +class _Nullable Smart; + +void f1(int * _Nonnull); + +void f2(Smart); // OK, not required on smart-pointer types +using Alias = Smart; +void f3(Alias); diff --git a/clang/test/SemaObjCXX/nullability-consistency.mm b/clang/test/SemaObjCXX/nullability-consistency.mm index 6921d8b9d3dd..09c9a84475a9 100644 --- a/clang/test/SemaObjCXX/nullability-consistency.mm +++ b/clang/test/SemaObjCXX/nullability-consistency.mm @@ -9,6 +9,7 @@ #include "nullability-consistency-6.h" #include "nullability-consistency-7.h" #include "nullability-consistency-8.h" +#include "nullability-consistency-smart.h" #include "nullability-consistency-system.h" void h1(int *ptr) { } // don't warn -- GitLab From beeb15b71650b46f39cb6b1917e8d05568978656 Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Tue, 2 Apr 2024 13:52:07 +0200 Subject: [PATCH 0185/1690] [libc++][NFC] Remove a few unused <__availablity> includes (#86126) --- libcxx/include/__exception/operations.h | 1 - libcxx/include/__filesystem/copy_options.h | 1 - libcxx/include/__filesystem/directory_options.h | 1 - libcxx/include/__filesystem/file_status.h | 1 - libcxx/include/__filesystem/file_time_type.h | 1 - libcxx/include/__filesystem/file_type.h | 1 - libcxx/include/__filesystem/perm_options.h | 1 - libcxx/include/__filesystem/perms.h | 1 - libcxx/include/__filesystem/space_info.h | 1 - libcxx/include/__format/format_args.h | 1 - libcxx/include/__format/format_context.h | 1 - libcxx/include/__format/formatter.h | 1 - libcxx/include/__format/formatter_bool.h | 1 - libcxx/include/__format/formatter_char.h | 1 - libcxx/include/__format/formatter_integer.h | 1 - libcxx/include/__format/formatter_pointer.h | 1 - libcxx/include/__format/formatter_string.h | 1 - libcxx/include/__fwd/format.h | 1 - libcxx/include/__locale | 1 - libcxx/include/__memory/shared_ptr.h | 1 - libcxx/include/__thread/support/pthread.h | 1 - libcxx/include/any | 1 - libcxx/include/future | 1 - libcxx/include/new | 1 - libcxx/include/shared_mutex | 1 - libcxx/include/thread | 1 - libcxx/include/typeinfo | 1 - 27 files changed, 27 deletions(-) diff --git a/libcxx/include/__exception/operations.h b/libcxx/include/__exception/operations.h index 8f374c0ccee5..0a9c7a7c7f0d 100644 --- a/libcxx/include/__exception/operations.h +++ b/libcxx/include/__exception/operations.h @@ -9,7 +9,6 @@ #ifndef _LIBCPP___EXCEPTION_OPERATIONS_H #define _LIBCPP___EXCEPTION_OPERATIONS_H -#include <__availability> #include <__config> #include diff --git a/libcxx/include/__filesystem/copy_options.h b/libcxx/include/__filesystem/copy_options.h index 1bf71292c8a6..097eebe61137 100644 --- a/libcxx/include/__filesystem/copy_options.h +++ b/libcxx/include/__filesystem/copy_options.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FILESYSTEM_COPY_OPTIONS_H #define _LIBCPP___FILESYSTEM_COPY_OPTIONS_H -#include <__availability> #include <__config> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) diff --git a/libcxx/include/__filesystem/directory_options.h b/libcxx/include/__filesystem/directory_options.h index 683c4678e083..d0cd3ebfdaa7 100644 --- a/libcxx/include/__filesystem/directory_options.h +++ b/libcxx/include/__filesystem/directory_options.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H #define _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H -#include <__availability> #include <__config> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) diff --git a/libcxx/include/__filesystem/file_status.h b/libcxx/include/__filesystem/file_status.h index 3e2b32eef82e..da316c8b0274 100644 --- a/libcxx/include/__filesystem/file_status.h +++ b/libcxx/include/__filesystem/file_status.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FILESYSTEM_FILE_STATUS_H #define _LIBCPP___FILESYSTEM_FILE_STATUS_H -#include <__availability> #include <__config> #include <__filesystem/file_type.h> #include <__filesystem/perms.h> diff --git a/libcxx/include/__filesystem/file_time_type.h b/libcxx/include/__filesystem/file_time_type.h index e086dbcc3f51..63e4ae1578cf 100644 --- a/libcxx/include/__filesystem/file_time_type.h +++ b/libcxx/include/__filesystem/file_time_type.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H #define _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H -#include <__availability> #include <__chrono/file_clock.h> #include <__chrono/time_point.h> #include <__config> diff --git a/libcxx/include/__filesystem/file_type.h b/libcxx/include/__filesystem/file_type.h index c509085d90de..e4ac1dfee9ed 100644 --- a/libcxx/include/__filesystem/file_type.h +++ b/libcxx/include/__filesystem/file_type.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FILESYSTEM_FILE_TYPE_H #define _LIBCPP___FILESYSTEM_FILE_TYPE_H -#include <__availability> #include <__config> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) diff --git a/libcxx/include/__filesystem/perm_options.h b/libcxx/include/__filesystem/perm_options.h index 529ef13558e9..64c16ee60a17 100644 --- a/libcxx/include/__filesystem/perm_options.h +++ b/libcxx/include/__filesystem/perm_options.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FILESYSTEM_PERM_OPTIONS_H #define _LIBCPP___FILESYSTEM_PERM_OPTIONS_H -#include <__availability> #include <__config> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) diff --git a/libcxx/include/__filesystem/perms.h b/libcxx/include/__filesystem/perms.h index 8f5f9a7e8248..458f1e6e5348 100644 --- a/libcxx/include/__filesystem/perms.h +++ b/libcxx/include/__filesystem/perms.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FILESYSTEM_PERMS_H #define _LIBCPP___FILESYSTEM_PERMS_H -#include <__availability> #include <__config> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) diff --git a/libcxx/include/__filesystem/space_info.h b/libcxx/include/__filesystem/space_info.h index 2e80ae3b2c11..3fa57d33096f 100644 --- a/libcxx/include/__filesystem/space_info.h +++ b/libcxx/include/__filesystem/space_info.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FILESYSTEM_SPACE_INFO_H #define _LIBCPP___FILESYSTEM_SPACE_INFO_H -#include <__availability> #include <__config> #include diff --git a/libcxx/include/__format/format_args.h b/libcxx/include/__format/format_args.h index 79fe51f96c6a..a5fde36a2981 100644 --- a/libcxx/include/__format/format_args.h +++ b/libcxx/include/__format/format_args.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FORMAT_FORMAT_ARGS_H #define _LIBCPP___FORMAT_FORMAT_ARGS_H -#include <__availability> #include <__config> #include <__format/format_arg.h> #include <__format/format_arg_store.h> diff --git a/libcxx/include/__format/format_context.h b/libcxx/include/__format/format_context.h index bf603c5c62d9..087d4bf289b8 100644 --- a/libcxx/include/__format/format_context.h +++ b/libcxx/include/__format/format_context.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FORMAT_FORMAT_CONTEXT_H #define _LIBCPP___FORMAT_FORMAT_CONTEXT_H -#include <__availability> #include <__concepts/same_as.h> #include <__config> #include <__format/buffer.h> diff --git a/libcxx/include/__format/formatter.h b/libcxx/include/__format/formatter.h index 47e35789b817..e2f418f936ee 100644 --- a/libcxx/include/__format/formatter.h +++ b/libcxx/include/__format/formatter.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FORMAT_FORMATTER_H #define _LIBCPP___FORMAT_FORMATTER_H -#include <__availability> #include <__config> #include <__fwd/format.h> diff --git a/libcxx/include/__format/formatter_bool.h b/libcxx/include/__format/formatter_bool.h index 5e3daff7b3db..17dc69541e8f 100644 --- a/libcxx/include/__format/formatter_bool.h +++ b/libcxx/include/__format/formatter_bool.h @@ -12,7 +12,6 @@ #include <__algorithm/copy.h> #include <__assert> -#include <__availability> #include <__config> #include <__format/concepts.h> #include <__format/format_parse_context.h> diff --git a/libcxx/include/__format/formatter_char.h b/libcxx/include/__format/formatter_char.h index 3358d422252f..d33e84368a76 100644 --- a/libcxx/include/__format/formatter_char.h +++ b/libcxx/include/__format/formatter_char.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FORMAT_FORMATTER_CHAR_H #define _LIBCPP___FORMAT_FORMATTER_CHAR_H -#include <__availability> #include <__concepts/same_as.h> #include <__config> #include <__format/concepts.h> diff --git a/libcxx/include/__format/formatter_integer.h b/libcxx/include/__format/formatter_integer.h index d57082b3881b..41400f00478e 100644 --- a/libcxx/include/__format/formatter_integer.h +++ b/libcxx/include/__format/formatter_integer.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FORMAT_FORMATTER_INTEGER_H #define _LIBCPP___FORMAT_FORMATTER_INTEGER_H -#include <__availability> #include <__concepts/arithmetic.h> #include <__config> #include <__format/concepts.h> diff --git a/libcxx/include/__format/formatter_pointer.h b/libcxx/include/__format/formatter_pointer.h index 3373996ec3d5..6941343efd91 100644 --- a/libcxx/include/__format/formatter_pointer.h +++ b/libcxx/include/__format/formatter_pointer.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FORMAT_FORMATTER_POINTER_H #define _LIBCPP___FORMAT_FORMATTER_POINTER_H -#include <__availability> #include <__config> #include <__format/concepts.h> #include <__format/format_parse_context.h> diff --git a/libcxx/include/__format/formatter_string.h b/libcxx/include/__format/formatter_string.h index d1ccfb9b5f7d..347439fc8dff 100644 --- a/libcxx/include/__format/formatter_string.h +++ b/libcxx/include/__format/formatter_string.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FORMAT_FORMATTER_STRING_H #define _LIBCPP___FORMAT_FORMATTER_STRING_H -#include <__availability> #include <__config> #include <__format/concepts.h> #include <__format/format_parse_context.h> diff --git a/libcxx/include/__fwd/format.h b/libcxx/include/__fwd/format.h index 6f5c71243711..b30c220f8a04 100644 --- a/libcxx/include/__fwd/format.h +++ b/libcxx/include/__fwd/format.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___FWD_FORMAT_H #define _LIBCPP___FWD_FORMAT_H -#include <__availability> #include <__config> #include <__iterator/concepts.h> diff --git a/libcxx/include/__locale b/libcxx/include/__locale index 2186db849331..fab87f0d6a27 100644 --- a/libcxx/include/__locale +++ b/libcxx/include/__locale @@ -10,7 +10,6 @@ #ifndef _LIBCPP___LOCALE #define _LIBCPP___LOCALE -#include <__availability> #include <__config> #include <__locale_dir/locale_base_api.h> #include <__memory/shared_ptr.h> // __shared_count diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h index 794a794d8fd8..a8ff189df2aa 100644 --- a/libcxx/include/__memory/shared_ptr.h +++ b/libcxx/include/__memory/shared_ptr.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___MEMORY_SHARED_PTR_H #define _LIBCPP___MEMORY_SHARED_PTR_H -#include <__availability> #include <__compare/compare_three_way.h> #include <__compare/ordering.h> #include <__config> diff --git a/libcxx/include/__thread/support/pthread.h b/libcxx/include/__thread/support/pthread.h index e194e5c68ad3..531f3e71de83 100644 --- a/libcxx/include/__thread/support/pthread.h +++ b/libcxx/include/__thread/support/pthread.h @@ -10,7 +10,6 @@ #ifndef _LIBCPP___THREAD_SUPPORT_PTHREAD_H #define _LIBCPP___THREAD_SUPPORT_PTHREAD_H -#include <__availability> #include <__chrono/convert_to_timespec.h> #include <__chrono/duration.h> #include <__config> diff --git a/libcxx/include/any b/libcxx/include/any index a6212fedfa2c..0e66890593c3 100644 --- a/libcxx/include/any +++ b/libcxx/include/any @@ -80,7 +80,6 @@ namespace std { */ -#include <__availability> #include <__config> #include <__memory/allocator.h> #include <__memory/allocator_destructor.h> diff --git a/libcxx/include/future b/libcxx/include/future index fda1591818a6..3c228686063e 100644 --- a/libcxx/include/future +++ b/libcxx/include/future @@ -369,7 +369,6 @@ template struct uses_allocator, Alloc>; #endif #include <__assert> -#include <__availability> #include <__chrono/duration.h> #include <__chrono/time_point.h> #include <__exception/exception_ptr.h> diff --git a/libcxx/include/new b/libcxx/include/new index 988f7a84422c..5a245dc5ef45 100644 --- a/libcxx/include/new +++ b/libcxx/include/new @@ -86,7 +86,6 @@ void operator delete[](void* ptr, void*) noexcept; */ -#include <__availability> #include <__config> #include <__exception/exception.h> #include <__type_traits/is_function.h> diff --git a/libcxx/include/shared_mutex b/libcxx/include/shared_mutex index 38b559e8930f..9cc391db6fc5 100644 --- a/libcxx/include/shared_mutex +++ b/libcxx/include/shared_mutex @@ -128,7 +128,6 @@ template # error " is not supported since libc++ has been configured without support for threads." #endif -#include <__availability> #include <__chrono/duration.h> #include <__chrono/steady_clock.h> #include <__chrono/time_point.h> diff --git a/libcxx/include/thread b/libcxx/include/thread index ed70bde76094..68ce63bd0143 100644 --- a/libcxx/include/thread +++ b/libcxx/include/thread @@ -92,7 +92,6 @@ void sleep_for(const chrono::duration& rel_time); # error " is not supported since libc++ has been configured without support for threads." #endif -#include <__availability> #include <__thread/formatter.h> #include <__thread/jthread.h> #include <__thread/support.h> diff --git a/libcxx/include/typeinfo b/libcxx/include/typeinfo index dafc7b89248e..1ae075edd4b3 100644 --- a/libcxx/include/typeinfo +++ b/libcxx/include/typeinfo @@ -56,7 +56,6 @@ public: */ -#include <__availability> #include <__config> #include <__exception/exception.h> #include <__type_traits/is_constant_evaluated.h> -- GitLab From 2950283dddab03c183c1be2d7de9d4999cc86131 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Tue, 2 Apr 2024 08:14:04 -0400 Subject: [PATCH 0186/1690] [libc++] Simplify the implementation of (#86843) Libc++'s own is complicated by the need to handle various platform-specific macros and to support duplicate inclusion. In reality, we only need to add a declaration of nullptr_t to it, so we can simply include the underlying outside of our guards to let it handle re-inclusion itself. --- libcxx/include/stddef.h | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/libcxx/include/stddef.h b/libcxx/include/stddef.h index 887776b150e4..470b5408336c 100644 --- a/libcxx/include/stddef.h +++ b/libcxx/include/stddef.h @@ -7,18 +7,6 @@ // //===----------------------------------------------------------------------===// -#if defined(__need_ptrdiff_t) || defined(__need_size_t) || defined(__need_wchar_t) || defined(__need_NULL) || \ - defined(__need_wint_t) - -# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) -# pragma GCC system_header -# endif - -# include_next - -#elif !defined(_LIBCPP_STDDEF_H) -# define _LIBCPP_STDDEF_H - /* stddef.h synopsis @@ -36,16 +24,19 @@ Types: */ -# include <__config> +#include <__config> + +// Note: This include is outside of header guards because we sometimes get included multiple times +// with different defines and the underlying will know how to deal with that. +#include_next + +#ifndef _LIBCPP_STDDEF_H +# define _LIBCPP_STDDEF_H # if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header # endif -# if __has_include_next() -# include_next -# endif - # ifdef __cplusplus typedef decltype(nullptr) nullptr_t; # endif -- GitLab From a4798bb0b67533b37d6b34fd5292714aac3b17d9 Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Tue, 2 Apr 2024 14:29:29 +0200 Subject: [PATCH 0187/1690] [flang][NFC] use mlir::SymbolTable in lowering (#86673) Whenever lowering is checking if a function or global already exists in the mlir::Module, it was doing module->lookup. On big programs (~5000 globals and functions), this causes important slowdowns because these lookups are linear. Use mlir::SymbolTable to speed-up these lookups. The SymbolTable has to be created from the ModuleOp and maintained in sync. It is therefore placed in the converter, and FirOPBuilders can take a pointer to it to speed-up the lookups. This patch does not bring mlir::SymbolTable to FIR/HLFIR passes, but some passes creating a lot of runtime calls could benefit from it too. More analysis will be needed. As an example of the speed-ups, this patch speeds-up compilation of Whizard compare_amplitude_UFO.F90 from 5 mins to 2 mins on my machine (there is still room for speed-ups). --- flang/include/flang/Lower/AbstractConverter.h | 13 ++++ .../flang/Optimizer/Builder/FIRBuilder.h | 66 +++++++++---------- .../flang/Optimizer/Dialect/FIROpsSupport.h | 19 +++--- flang/lib/Lower/Bridge.cpp | 23 +++++-- flang/lib/Lower/CallInterface.cpp | 9 ++- flang/lib/Lower/OpenACC.cpp | 6 +- flang/lib/Optimizer/Builder/FIRBuilder.cpp | 60 +++++++++++++---- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 6 +- .../Optimizer/Builder/LowLevelIntrinsics.cpp | 62 +++++++++-------- .../Optimizer/Builder/PPCIntrinsicCall.cpp | 34 +++++----- flang/lib/Optimizer/Dialect/FIROps.cpp | 28 ++++++-- .../Transforms/SimplifyIntrinsics.cpp | 7 +- 12 files changed, 206 insertions(+), 127 deletions(-) diff --git a/flang/include/flang/Lower/AbstractConverter.h b/flang/include/flang/Lower/AbstractConverter.h index 32e7a5e2b040..d5dab9040d22 100644 --- a/flang/include/flang/Lower/AbstractConverter.h +++ b/flang/include/flang/Lower/AbstractConverter.h @@ -23,6 +23,10 @@ #include "mlir/IR/Operation.h" #include "llvm/ADT/ArrayRef.h" +namespace mlir { +class SymbolTable; +} + namespace fir { class KindMapping; class FirOpBuilder; @@ -305,6 +309,15 @@ public: virtual Fortran::lower::SymbolBox lookupOneLevelUpSymbol(const Fortran::semantics::Symbol &sym) = 0; + /// Return the mlir::SymbolTable associated to the ModuleOp. + /// Look-ups are faster using it than using module.lookup<>, + /// but the module op should be queried in case of failure + /// because this symbol table is not guaranteed to contain + /// all the symbols from the ModuleOp (the symbol table should + /// always be provided to the builder helper creating globals and + /// functions in order to be in sync). + virtual mlir::SymbolTable *getMLIRSymbolTable() = 0; + private: /// Options controlling lowering behavior. const Fortran::lower::LoweringOptions &loweringOptions; diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h index d61bf681be61..940866b25d2f 100644 --- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h +++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h @@ -28,6 +28,10 @@ #include #include +namespace mlir { +class SymbolTable; +} + namespace fir { class AbstractArrayBox; class ExtendedValue; @@ -42,8 +46,10 @@ class BoxValue; /// patterns. class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener { public: - explicit FirOpBuilder(mlir::Operation *op, fir::KindMapping kindMap) - : OpBuilder{op, /*listener=*/this}, kindMap{std::move(kindMap)} {} + explicit FirOpBuilder(mlir::Operation *op, fir::KindMapping kindMap, + mlir::SymbolTable *symbolTable = nullptr) + : OpBuilder{op, /*listener=*/this}, kindMap{std::move(kindMap)}, + symbolTable{symbolTable} {} explicit FirOpBuilder(mlir::OpBuilder &builder, fir::KindMapping kindMap) : OpBuilder(builder), OpBuilder::Listener(), kindMap{std::move(kindMap)} { setListener(this); @@ -69,13 +75,14 @@ public: // The listener self-reference has to be updated in case of copy-construction. FirOpBuilder(const FirOpBuilder &other) : OpBuilder(other), OpBuilder::Listener(), kindMap{other.kindMap}, - fastMathFlags{other.fastMathFlags} { + fastMathFlags{other.fastMathFlags}, symbolTable{other.symbolTable} { setListener(this); } FirOpBuilder(FirOpBuilder &&other) : OpBuilder(other), OpBuilder::Listener(), - kindMap{std::move(other.kindMap)}, fastMathFlags{other.fastMathFlags} { + kindMap{std::move(other.kindMap)}, fastMathFlags{other.fastMathFlags}, + symbolTable{other.symbolTable} { setListener(this); } @@ -95,6 +102,9 @@ public: /// Get a reference to the kind map. const fir::KindMapping &getKindMap() { return kindMap; } + /// Get func.func/fir.global symbol table attached to this builder if any. + mlir::SymbolTable *getMLIRSymbolTable() { return symbolTable; } + /// Get the default integer type [[maybe_unused]] mlir::IntegerType getDefaultIntegerType() { return getIntegerType( @@ -280,24 +290,27 @@ public: /// Get a function by name. If the function exists in the current module, it /// is returned. Otherwise, a null FuncOp is returned. mlir::func::FuncOp getNamedFunction(llvm::StringRef name) { - return getNamedFunction(getModule(), name); + return getNamedFunction(getModule(), getMLIRSymbolTable(), name); } - static mlir::func::FuncOp getNamedFunction(mlir::ModuleOp module, - llvm::StringRef name); + static mlir::func::FuncOp + getNamedFunction(mlir::ModuleOp module, const mlir::SymbolTable *symbolTable, + llvm::StringRef name); /// Get a function by symbol name. The result will be null if there is no /// function with the given symbol in the module. mlir::func::FuncOp getNamedFunction(mlir::SymbolRefAttr symbol) { - return getNamedFunction(getModule(), symbol); + return getNamedFunction(getModule(), getMLIRSymbolTable(), symbol); } - static mlir::func::FuncOp getNamedFunction(mlir::ModuleOp module, - mlir::SymbolRefAttr symbol); + static mlir::func::FuncOp + getNamedFunction(mlir::ModuleOp module, const mlir::SymbolTable *symbolTable, + mlir::SymbolRefAttr symbol); fir::GlobalOp getNamedGlobal(llvm::StringRef name) { - return getNamedGlobal(getModule(), name); + return getNamedGlobal(getModule(), getMLIRSymbolTable(), name); } static fir::GlobalOp getNamedGlobal(mlir::ModuleOp module, + const mlir::SymbolTable *symbolTable, llvm::StringRef name); /// Lazy creation of fir.convert op. @@ -313,35 +326,18 @@ public: /// result of the load if it was created, otherwise return \p val mlir::Value loadIfRef(mlir::Location loc, mlir::Value val); - /// Create a new FuncOp. If the function may have already been created, use - /// `addNamedFunction` instead. + /// Determine if the named function is already in the module. Return the + /// instance if found, otherwise add a new named function to the module. mlir::func::FuncOp createFunction(mlir::Location loc, llvm::StringRef name, mlir::FunctionType ty) { - return createFunction(loc, getModule(), name, ty); + return createFunction(loc, getModule(), name, ty, getMLIRSymbolTable()); } static mlir::func::FuncOp createFunction(mlir::Location loc, mlir::ModuleOp module, llvm::StringRef name, - mlir::FunctionType ty); - - /// Determine if the named function is already in the module. Return the - /// instance if found, otherwise add a new named function to the module. - mlir::func::FuncOp addNamedFunction(mlir::Location loc, llvm::StringRef name, - mlir::FunctionType ty) { - if (auto func = getNamedFunction(name)) - return func; - return createFunction(loc, name, ty); - } - - static mlir::func::FuncOp addNamedFunction(mlir::Location loc, - mlir::ModuleOp module, - llvm::StringRef name, - mlir::FunctionType ty) { - if (auto func = getNamedFunction(module, name)) - return func; - return createFunction(loc, module, name, ty); - } + mlir::FunctionType ty, + mlir::SymbolTable *); /// Cast the input value to IndexType. mlir::Value convertToIndexType(mlir::Location loc, mlir::Value val) { @@ -515,6 +511,10 @@ private: /// FastMathFlags that need to be set for operations that support /// mlir::arith::FastMathAttr. mlir::arith::FastMathFlags fastMathFlags{}; + + /// fir::GlobalOp and func::FuncOp symbol table to speed-up + /// lookups. + mlir::SymbolTable *symbolTable = nullptr; }; } // namespace fir diff --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h index e8226b6df58c..f29e44504acb 100644 --- a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h +++ b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h @@ -52,16 +52,19 @@ inline bool pureCall(mlir::Operation *op) { /// Get or create a FuncOp in a module. /// /// If `module` already contains FuncOp `name`, it is returned. Otherwise, a new -/// FuncOp is created, and that new FuncOp is returned. -mlir::func::FuncOp -createFuncOp(mlir::Location loc, mlir::ModuleOp module, llvm::StringRef name, - mlir::FunctionType type, - llvm::ArrayRef attrs = {}); - -/// Get or create a GlobalOp in a module. +/// FuncOp is created, and that new FuncOp is returned. A symbol table can +/// be provided to speed-up the lookups. +mlir::func::FuncOp createFuncOp(mlir::Location loc, mlir::ModuleOp module, + llvm::StringRef name, mlir::FunctionType type, + llvm::ArrayRef attrs = {}, + const mlir::SymbolTable *symbolTable = nullptr); + +/// Get or create a GlobalOp in a module. A symbol table can be provided to +/// speed-up the lookups. fir::GlobalOp createGlobalOp(mlir::Location loc, mlir::ModuleOp module, llvm::StringRef name, mlir::Type type, - llvm::ArrayRef attrs = {}); + llvm::ArrayRef attrs = {}, + const mlir::SymbolTable *symbolTable = nullptr); /// Attribute to mark Fortran entities with the CONTIGUOUS attribute. constexpr llvm::StringRef getContiguousAttrName() { return "fir.contiguous"; } diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp index 91b898eb513e..5bba0978617c 100644 --- a/flang/lib/Lower/Bridge.cpp +++ b/flang/lib/Lower/Bridge.cpp @@ -273,7 +273,8 @@ class FirConverter : public Fortran::lower::AbstractConverter { public: explicit FirConverter(Fortran::lower::LoweringBridge &bridge) : Fortran::lower::AbstractConverter(bridge.getLoweringOptions()), - bridge{bridge}, foldingContext{bridge.createFoldingContext()} {} + bridge{bridge}, foldingContext{bridge.createFoldingContext()}, + mlirSymbolTable{bridge.getModule()} {} virtual ~FirConverter() = default; /// Convert the PFT to FIR. @@ -329,8 +330,8 @@ public: [&](Fortran::lower::pft::BlockDataUnit &b) {}, [&](Fortran::lower::pft::CompilerDirectiveUnit &d) {}, [&](Fortran::lower::pft::OpenACCDirectiveUnit &d) { - builder = new fir::FirOpBuilder(bridge.getModule(), - bridge.getKindMap()); + builder = new fir::FirOpBuilder( + bridge.getModule(), bridge.getKindMap(), &mlirSymbolTable); Fortran::lower::genOpenACCRoutineConstruct( *this, bridge.getSemanticsContext(), bridge.getModule(), d.routine, accRoutineInfos); @@ -1036,6 +1037,8 @@ private: return {}; } + mlir::SymbolTable *getMLIRSymbolTable() override { return &mlirSymbolTable; } + /// Add the symbol to the local map and return `true`. If the symbol is /// already in the map and \p forced is `false`, the map is not updated. /// Instead the value `false` is returned. @@ -4571,7 +4574,8 @@ private: llvm::dbgs() << "\n"); Fortran::lower::CalleeInterface callee(funit, *this); mlir::func::FuncOp func = callee.addEntryBlockAndMapArguments(); - builder = new fir::FirOpBuilder(func, bridge.getKindMap()); + builder = + new fir::FirOpBuilder(func, bridge.getKindMap(), &mlirSymbolTable); assert(builder && "FirOpBuilder did not instantiate"); builder->setFastMathFlags(bridge.getLoweringOptions().getMathOptions()); builder->setInsertionPointToStart(&func.front()); @@ -4839,12 +4843,14 @@ private: // FIXME: get rid of the bogus function context and instantiate the // globals directly into the module. mlir::MLIRContext *context = &getMLIRContext(); + mlir::SymbolTable *symbolTable = getMLIRSymbolTable(); mlir::func::FuncOp func = fir::FirOpBuilder::createFunction( mlir::UnknownLoc::get(context), getModuleOp(), fir::NameUniquer::doGenerated("Sham"), - mlir::FunctionType::get(context, std::nullopt, std::nullopt)); + mlir::FunctionType::get(context, std::nullopt, std::nullopt), + symbolTable); func.addEntryBlock(); - builder = new fir::FirOpBuilder(func, bridge.getKindMap()); + builder = new fir::FirOpBuilder(func, bridge.getKindMap(), symbolTable); assert(builder && "FirOpBuilder did not instantiate"); builder->setFastMathFlags(bridge.getLoweringOptions().getMathOptions()); createGlobals(); @@ -5336,6 +5342,11 @@ private: /// utilities to deal with procedure pointer components whose arguments have /// the type of the containing derived type. Fortran::lower::TypeConstructionStack typeConstructionStack; + /// MLIR symbol table of the fir.global/func.func operations. Note that it is + /// not guaranteed to contain all operations of the ModuleOp with Symbol + /// attribute since mlirSymbolTable must pro-actively be maintained when + /// new Symbol operations are created. + mlir::SymbolTable mlirSymbolTable; }; } // namespace diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp index c65becc49745..29cdb3cff589 100644 --- a/flang/lib/Lower/CallInterface.cpp +++ b/flang/lib/Lower/CallInterface.cpp @@ -667,11 +667,13 @@ void Fortran::lower::CallInterface::declare() { if (!side().isIndirectCall()) { std::string name = side().getMangledName(); mlir::ModuleOp module = converter.getModuleOp(); - func = fir::FirOpBuilder::getNamedFunction(module, name); + mlir::SymbolTable *symbolTable = converter.getMLIRSymbolTable(); + func = fir::FirOpBuilder::getNamedFunction(module, symbolTable, name); if (!func) { mlir::Location loc = side().getCalleeLocation(); mlir::FunctionType ty = genFunctionType(); - func = fir::FirOpBuilder::createFunction(loc, module, name, ty); + func = + fir::FirOpBuilder::createFunction(loc, module, name, ty, symbolTable); if (const Fortran::semantics::Symbol *sym = side().getProcedureSymbol()) { if (side().isMainProgram()) { func->setAttr(fir::getSymbolAttrName(), @@ -1644,7 +1646,8 @@ mlir::func::FuncOp Fortran::lower::getOrDeclareFunction( Fortran::lower::AbstractConverter &converter) { mlir::ModuleOp module = converter.getModuleOp(); std::string name = getProcMangledName(proc, converter); - mlir::func::FuncOp func = fir::FirOpBuilder::getNamedFunction(module, name); + mlir::func::FuncOp func = fir::FirOpBuilder::getNamedFunction( + module, converter.getMLIRSymbolTable(), name); if (func) return func; diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp index 6e6714454f05..d933c07aba0e 100644 --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -3821,7 +3821,8 @@ void Fortran::lower::genOpenACCRoutineConstruct( std::string funcName; if (name) { funcName = converter.mangleName(*name->symbol); - funcOp = builder.getNamedFunction(mod, funcName); + funcOp = + builder.getNamedFunction(mod, builder.getMLIRSymbolTable(), funcName); } else { Fortran::semantics::Scope &scope = semanticsContext.FindScope(routineConstruct.source); @@ -3833,7 +3834,8 @@ void Fortran::lower::genOpenACCRoutineConstruct( : nullptr}; if (subpDetails && subpDetails->isInterface()) { funcName = converter.mangleName(*progUnit.symbol()); - funcOp = builder.getNamedFunction(mod, funcName); + funcOp = + builder.getNamedFunction(mod, builder.getMLIRSymbolTable(), funcName); } else { funcOp = builder.getFunction(); funcName = funcOp.getName(); diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp index 2bcd5e591402..e4362b2f9e69 100644 --- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp +++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp @@ -36,26 +36,56 @@ static llvm::cl::opt "name"), llvm::cl::init(32)); -mlir::func::FuncOp fir::FirOpBuilder::createFunction(mlir::Location loc, - mlir::ModuleOp module, - llvm::StringRef name, - mlir::FunctionType ty) { - return fir::createFuncOp(loc, module, name, ty); +mlir::func::FuncOp +fir::FirOpBuilder::createFunction(mlir::Location loc, mlir::ModuleOp module, + llvm::StringRef name, mlir::FunctionType ty, + mlir::SymbolTable *symbolTable) { + return fir::createFuncOp(loc, module, name, ty, /*attrs*/ {}, symbolTable); } -mlir::func::FuncOp fir::FirOpBuilder::getNamedFunction(mlir::ModuleOp modOp, - llvm::StringRef name) { +mlir::func::FuncOp +fir::FirOpBuilder::getNamedFunction(mlir::ModuleOp modOp, + const mlir::SymbolTable *symbolTable, + llvm::StringRef name) { + if (symbolTable) + if (auto func = symbolTable->lookup(name)) { +#ifdef EXPENSIVE_CHECKS + assert(func == modOp.lookupSymbol(name) && + "symbolTable and module out of sync"); +#endif + return func; + } return modOp.lookupSymbol(name); } mlir::func::FuncOp fir::FirOpBuilder::getNamedFunction(mlir::ModuleOp modOp, + const mlir::SymbolTable *symbolTable, mlir::SymbolRefAttr symbol) { + if (symbolTable) + if (auto func = symbolTable->lookup( + symbol.getLeafReference())) { +#ifdef EXPENSIVE_CHECKS + assert(func == modOp.lookupSymbol(symbol) && + "symbolTable and module out of sync"); +#endif + return func; + } return modOp.lookupSymbol(symbol); } -fir::GlobalOp fir::FirOpBuilder::getNamedGlobal(mlir::ModuleOp modOp, - llvm::StringRef name) { +fir::GlobalOp +fir::FirOpBuilder::getNamedGlobal(mlir::ModuleOp modOp, + const mlir::SymbolTable *symbolTable, + llvm::StringRef name) { + if (symbolTable) + if (auto global = symbolTable->lookup(name)) { +#ifdef EXPENSIVE_CHECKS + assert(global == modOp.lookupSymbol(name) && + "symbolTable and module out of sync"); +#endif + return global; + } return modOp.lookupSymbol(name); } @@ -279,10 +309,10 @@ fir::GlobalOp fir::FirOpBuilder::createGlobal( mlir::Location loc, mlir::Type type, llvm::StringRef name, mlir::StringAttr linkage, mlir::Attribute value, bool isConst, bool isTarget, fir::CUDADataAttributeAttr cudaAttr) { + if (auto global = getNamedGlobal(name)) + return global; auto module = getModule(); auto insertPt = saveInsertionPoint(); - if (auto glob = module.lookupSymbol(name)) - return glob; setInsertionPoint(module.getBody(), module.getBody()->end()); llvm::SmallVector attrs; if (cudaAttr) { @@ -294,6 +324,8 @@ fir::GlobalOp fir::FirOpBuilder::createGlobal( auto glob = create(loc, name, isConst, isTarget, type, value, linkage, attrs); restoreInsertionPoint(insertPt); + if (symbolTable) + symbolTable->insert(glob); return glob; } @@ -301,10 +333,10 @@ fir::GlobalOp fir::FirOpBuilder::createGlobal( mlir::Location loc, mlir::Type type, llvm::StringRef name, bool isConst, bool isTarget, std::function bodyBuilder, mlir::StringAttr linkage, fir::CUDADataAttributeAttr cudaAttr) { + if (auto global = getNamedGlobal(name)) + return global; auto module = getModule(); auto insertPt = saveInsertionPoint(); - if (auto glob = module.lookupSymbol(name)) - return glob; setInsertionPoint(module.getBody(), module.getBody()->end()); auto glob = create(loc, name, isConst, isTarget, type, mlir::Attribute{}, linkage); @@ -314,6 +346,8 @@ fir::GlobalOp fir::FirOpBuilder::createGlobal( setInsertionPointToStart(&block); bodyBuilder(*this); restoreInsertionPoint(insertPt); + if (symbolTable) + symbolTable->insert(glob); return glob; } diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index ea1ef1f08aba..069ba81cfe96 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -724,7 +724,7 @@ mlir::Value genLibCall(fir::FirOpBuilder &builder, mlir::Location loc, mlir::func::FuncOp funcOp = builder.getNamedFunction(libFuncName); if (!funcOp) { - funcOp = builder.addNamedFunction(loc, libFuncName, libFuncType); + funcOp = builder.createFunction(loc, libFuncName, libFuncType); // C-interoperability rules apply to these library functions. funcOp->setAttr(fir::getSymbolAttrName(), mlir::StringAttr::get(builder.getContext(), libFuncName)); @@ -1894,8 +1894,8 @@ mlir::func::FuncOp IntrinsicLibrary::getWrapper(GeneratorType generator, // Create local context to emit code into the newly created function // This new function is not linked to a source file location, only // its calls will be. - auto localBuilder = - std::make_unique(function, builder.getKindMap()); + auto localBuilder = std::make_unique( + function, builder.getKindMap(), builder.getMLIRSymbolTable()); localBuilder->setFastMathFlags(builder.getFastMathFlags()); localBuilder->setInsertionPointToStart(&function.front()); // Location of code inside wrapper of the wrapper is independent from diff --git a/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp b/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp index 1d07b1e724d7..bb5f77d5d4d1 100644 --- a/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp +++ b/flang/lib/Optimizer/Builder/LowLevelIntrinsics.cpp @@ -27,8 +27,8 @@ mlir::func::FuncOp fir::factory::getLlvmMemcpy(fir::FirOpBuilder &builder) { builder.getI1Type()}; auto memcpyTy = mlir::FunctionType::get(builder.getContext(), args, std::nullopt); - return builder.addNamedFunction(builder.getUnknownLoc(), - "llvm.memcpy.p0.p0.i64", memcpyTy); + return builder.createFunction(builder.getUnknownLoc(), + "llvm.memcpy.p0.p0.i64", memcpyTy); } mlir::func::FuncOp fir::factory::getLlvmMemmove(fir::FirOpBuilder &builder) { @@ -37,8 +37,8 @@ mlir::func::FuncOp fir::factory::getLlvmMemmove(fir::FirOpBuilder &builder) { builder.getI1Type()}; auto memmoveTy = mlir::FunctionType::get(builder.getContext(), args, std::nullopt); - return builder.addNamedFunction(builder.getUnknownLoc(), - "llvm.memmove.p0.p0.i64", memmoveTy); + return builder.createFunction(builder.getUnknownLoc(), + "llvm.memmove.p0.p0.i64", memmoveTy); } mlir::func::FuncOp fir::factory::getLlvmMemset(fir::FirOpBuilder &builder) { @@ -47,16 +47,15 @@ mlir::func::FuncOp fir::factory::getLlvmMemset(fir::FirOpBuilder &builder) { builder.getI1Type()}; auto memsetTy = mlir::FunctionType::get(builder.getContext(), args, std::nullopt); - return builder.addNamedFunction(builder.getUnknownLoc(), - "llvm.memset.p0.p0.i64", memsetTy); + return builder.createFunction(builder.getUnknownLoc(), + "llvm.memset.p0.p0.i64", memsetTy); } mlir::func::FuncOp fir::factory::getRealloc(fir::FirOpBuilder &builder) { auto ptrTy = builder.getRefType(builder.getIntegerType(8)); llvm::SmallVector args = {ptrTy, builder.getI64Type()}; auto reallocTy = mlir::FunctionType::get(builder.getContext(), args, {ptrTy}); - return builder.addNamedFunction(builder.getUnknownLoc(), "realloc", - reallocTy); + return builder.createFunction(builder.getUnknownLoc(), "realloc", reallocTy); } mlir::func::FuncOp @@ -64,8 +63,8 @@ fir::factory::getLlvmGetRounding(fir::FirOpBuilder &builder) { auto int32Ty = builder.getIntegerType(32); auto funcTy = mlir::FunctionType::get(builder.getContext(), std::nullopt, {int32Ty}); - return builder.addNamedFunction(builder.getUnknownLoc(), "llvm.get.rounding", - funcTy); + return builder.createFunction(builder.getUnknownLoc(), "llvm.get.rounding", + funcTy); } mlir::func::FuncOp @@ -73,8 +72,8 @@ fir::factory::getLlvmSetRounding(fir::FirOpBuilder &builder) { auto int32Ty = builder.getIntegerType(32); auto funcTy = mlir::FunctionType::get(builder.getContext(), {int32Ty}, std::nullopt); - return builder.addNamedFunction(builder.getUnknownLoc(), "llvm.set.rounding", - funcTy); + return builder.createFunction(builder.getUnknownLoc(), "llvm.set.rounding", + funcTy); } mlir::func::FuncOp fir::factory::getLlvmStackSave(fir::FirOpBuilder &builder) { @@ -82,8 +81,8 @@ mlir::func::FuncOp fir::factory::getLlvmStackSave(fir::FirOpBuilder &builder) { auto ptrTy = builder.getRefType(builder.getIntegerType(8)); auto funcTy = mlir::FunctionType::get(builder.getContext(), std::nullopt, {ptrTy}); - return builder.addNamedFunction(builder.getUnknownLoc(), "llvm.stacksave.p0", - funcTy); + return builder.createFunction(builder.getUnknownLoc(), "llvm.stacksave.p0", + funcTy); } mlir::func::FuncOp @@ -92,8 +91,8 @@ fir::factory::getLlvmStackRestore(fir::FirOpBuilder &builder) { auto ptrTy = builder.getRefType(builder.getIntegerType(8)); auto funcTy = mlir::FunctionType::get(builder.getContext(), {ptrTy}, std::nullopt); - return builder.addNamedFunction(builder.getUnknownLoc(), - "llvm.stackrestore.p0", funcTy); + return builder.createFunction(builder.getUnknownLoc(), "llvm.stackrestore.p0", + funcTy); } mlir::func::FuncOp @@ -101,24 +100,24 @@ fir::factory::getLlvmInitTrampoline(fir::FirOpBuilder &builder) { auto ptrTy = builder.getRefType(builder.getIntegerType(8)); auto funcTy = mlir::FunctionType::get(builder.getContext(), {ptrTy, ptrTy, ptrTy}, std::nullopt); - return builder.addNamedFunction(builder.getUnknownLoc(), - "llvm.init.trampoline", funcTy); + return builder.createFunction(builder.getUnknownLoc(), "llvm.init.trampoline", + funcTy); } mlir::func::FuncOp fir::factory::getLlvmAdjustTrampoline(fir::FirOpBuilder &builder) { auto ptrTy = builder.getRefType(builder.getIntegerType(8)); auto funcTy = mlir::FunctionType::get(builder.getContext(), {ptrTy}, {ptrTy}); - return builder.addNamedFunction(builder.getUnknownLoc(), - "llvm.adjust.trampoline", funcTy); + return builder.createFunction(builder.getUnknownLoc(), + "llvm.adjust.trampoline", funcTy); } mlir::func::FuncOp fir::factory::getFeclearexcept(fir::FirOpBuilder &builder) { auto int32Ty = builder.getIntegerType(32); auto funcTy = mlir::FunctionType::get(builder.getContext(), {int32Ty}, {int32Ty}); - return builder.addNamedFunction(builder.getUnknownLoc(), "feclearexcept", - funcTy); + return builder.createFunction(builder.getUnknownLoc(), "feclearexcept", + funcTy); } mlir::func::FuncOp @@ -126,38 +125,37 @@ fir::factory::getFedisableexcept(fir::FirOpBuilder &builder) { auto int32Ty = builder.getIntegerType(32); auto funcTy = mlir::FunctionType::get(builder.getContext(), {int32Ty}, {int32Ty}); - return builder.addNamedFunction(builder.getUnknownLoc(), "fedisableexcept", - funcTy); + return builder.createFunction(builder.getUnknownLoc(), "fedisableexcept", + funcTy); } mlir::func::FuncOp fir::factory::getFeenableexcept(fir::FirOpBuilder &builder) { auto int32Ty = builder.getIntegerType(32); auto funcTy = mlir::FunctionType::get(builder.getContext(), {int32Ty}, {int32Ty}); - return builder.addNamedFunction(builder.getUnknownLoc(), "feenableexcept", - funcTy); + return builder.createFunction(builder.getUnknownLoc(), "feenableexcept", + funcTy); } mlir::func::FuncOp fir::factory::getFegetexcept(fir::FirOpBuilder &builder) { auto int32Ty = builder.getIntegerType(32); auto funcTy = mlir::FunctionType::get(builder.getContext(), std::nullopt, {int32Ty}); - return builder.addNamedFunction(builder.getUnknownLoc(), "fegetexcept", - funcTy); + return builder.createFunction(builder.getUnknownLoc(), "fegetexcept", funcTy); } mlir::func::FuncOp fir::factory::getFeraiseexcept(fir::FirOpBuilder &builder) { auto int32Ty = builder.getIntegerType(32); auto funcTy = mlir::FunctionType::get(builder.getContext(), {int32Ty}, {int32Ty}); - return builder.addNamedFunction(builder.getUnknownLoc(), "feraiseexcept", - funcTy); + return builder.createFunction(builder.getUnknownLoc(), "feraiseexcept", + funcTy); } mlir::func::FuncOp fir::factory::getFetestexcept(fir::FirOpBuilder &builder) { auto int32Ty = builder.getIntegerType(32); auto funcTy = mlir::FunctionType::get(builder.getContext(), {int32Ty}, {int32Ty}); - return builder.addNamedFunction(builder.getUnknownLoc(), "fetestexcept", - funcTy); + return builder.createFunction(builder.getUnknownLoc(), "fetestexcept", + funcTy); } diff --git a/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp b/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp index ab0d5079d8af..e588b19dded4 100644 --- a/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp @@ -1084,11 +1084,11 @@ void PPCIntrinsicLibrary::genMtfsf(llvm::ArrayRef args) { if (isImm) { libFuncType = genFuncType, Ty::Integer<4>>( builder.getContext(), builder); - funcOp = builder.addNamedFunction(loc, "llvm.ppc.mtfsfi", libFuncType); + funcOp = builder.createFunction(loc, "llvm.ppc.mtfsfi", libFuncType); } else { libFuncType = genFuncType, Ty::Real<8>>( builder.getContext(), builder); - funcOp = builder.addNamedFunction(loc, "llvm.ppc.mtfsf", libFuncType); + funcOp = builder.createFunction(loc, "llvm.ppc.mtfsf", libFuncType); } builder.create(loc, funcOp, scalarArgs); } @@ -1116,7 +1116,7 @@ PPCIntrinsicLibrary::genVecAbs(mlir::Type resultType, genFuncType, Ty::RealVector<8>>(context, builder); } - funcOp = builder.addNamedFunction(loc, fname, ftype); + funcOp = builder.createFunction(loc, fname, ftype); auto callOp{builder.create(loc, funcOp, argBases[0])}; return callOp.getResult(0); } else if (auto eleTy = vTypeInfo.eleTy.dyn_cast()) { @@ -1155,7 +1155,7 @@ PPCIntrinsicLibrary::genVecAbs(mlir::Type resultType, default: llvm_unreachable("invalid integer size"); } - funcOp = builder.addNamedFunction(loc, fname, ftype); + funcOp = builder.createFunction(loc, fname, ftype); mlir::Value args[] = {zeroSubVarg1, varg1}; auto callOp{builder.create(loc, funcOp, args)}; @@ -1339,7 +1339,7 @@ PPCIntrinsicLibrary::genVecAnyCompare(mlir::Type resultType, } assert((!fname.empty() && ftype) && "invalid type"); - mlir::func::FuncOp funcOp{builder.addNamedFunction(loc, fname, ftype)}; + mlir::func::FuncOp funcOp{builder.createFunction(loc, fname, ftype)}; auto callOp{builder.create(loc, funcOp, cmpArgs)}; return callOp.getResult(0); } @@ -1445,7 +1445,7 @@ PPCIntrinsicLibrary::genVecCmp(mlir::Type resultType, std::pair funcTyNam{ getVecCmpFuncTypeAndName(vecTyInfo, vop, builder)}; - mlir::func::FuncOp funcOp = builder.addNamedFunction( + mlir::func::FuncOp funcOp = builder.createFunction( loc, std::get<0>(funcTyNam), std::get<1>(funcTyNam)); mlir::Value res{nullptr}; @@ -1572,7 +1572,7 @@ PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType, Ty::Integer<4>>(context, builder)}; const llvm::StringRef fname{(isUnsigned) ? "llvm.ppc.altivec.vcfux" : "llvm.ppc.altivec.vcfsx"}; - auto funcOp{builder.addNamedFunction(loc, fname, ftype)}; + auto funcOp{builder.createFunction(loc, fname, ftype)}; mlir::Value newArgs[] = {argBases[0], convArg}; auto callOp{builder.create(loc, funcOp, newArgs)}; @@ -1627,7 +1627,7 @@ PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType, const llvm::StringRef fname{"llvm.ppc.vsx.xvcvspdp"}; auto ftype{ genFuncType, Ty::RealVector<4>>(context, builder)}; - auto funcOp{builder.addNamedFunction(loc, fname, ftype)}; + auto funcOp{builder.createFunction(loc, fname, ftype)}; auto callOp{builder.create(loc, funcOp, newArgs)}; return callOp.getResult(0); @@ -1635,7 +1635,7 @@ PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType, const llvm::StringRef fname{"llvm.ppc.vsx.xvcvdpsp"}; auto ftype{ genFuncType, Ty::RealVector<8>>(context, builder)}; - auto funcOp{builder.addNamedFunction(loc, fname, ftype)}; + auto funcOp{builder.createFunction(loc, fname, ftype)}; newArgs[0] = builder.create(loc, funcOp, newArgs).getResult(0); auto fvf32Ty{newArgs[0].getType()}; @@ -1963,7 +1963,7 @@ PPCIntrinsicLibrary::genVecLdCallGrp(mlir::Type resultType, auto funcType{ mlir::FunctionType::get(context, {addr.getType()}, {intrinResTy})}; - auto funcOp{builder.addNamedFunction(loc, fname, funcType)}; + auto funcOp{builder.createFunction(loc, fname, funcType)}; auto result{ builder.create(loc, funcOp, parsedArgs).getResult(0)}; @@ -2022,7 +2022,7 @@ PPCIntrinsicLibrary::genVecLvsGrp(mlir::Type resultType, llvm_unreachable("invalid vector operation for generator"); } auto funcType{mlir::FunctionType::get(context, {addr.getType()}, {mlirTy})}; - auto funcOp{builder.addNamedFunction(loc, fname, funcType)}; + auto funcOp{builder.createFunction(loc, fname, funcType)}; auto result{ builder.create(loc, funcOp, parsedArgs).getResult(0)}; @@ -2057,8 +2057,8 @@ PPCIntrinsicLibrary::genVecNmaddMsub(mlir::Type resultType, genFuncType, Ty::RealVector<8>, Ty::RealVector<8>>( context, builder))}}; - auto funcOp{builder.addNamedFunction(loc, std::get<0>(fmaMap[width]), - std::get<1>(fmaMap[width]))}; + auto funcOp{builder.createFunction(loc, std::get<0>(fmaMap[width]), + std::get<1>(fmaMap[width]))}; if (vop == VecOp::Nmadd) { // vec_nmadd(arg1, arg2, arg3) = -fma(arg1, arg2, arg3) auto callOp{builder.create(loc, funcOp, newArgs)}; @@ -2110,7 +2110,7 @@ PPCIntrinsicLibrary::genVecPerm(mlir::Type resultType, builder.create(loc, vi32Ty, mArg1).getResult(); } - auto funcOp{builder.addNamedFunction( + auto funcOp{builder.createFunction( loc, "llvm.ppc.altivec.vperm", genFuncType, Ty::IntegerVector<4>, Ty::IntegerVector<4>, Ty::IntegerVector<1>>(context, @@ -2307,7 +2307,7 @@ PPCIntrinsicLibrary::genVecShift(mlir::Type resultType, } auto funcTy{genFuncType, Ty::IntegerVector<4>, Ty::IntegerVector<4>>(context, builder)}; - mlir::func::FuncOp funcOp{builder.addNamedFunction(loc, funcName, funcTy)}; + mlir::func::FuncOp funcOp{builder.createFunction(loc, funcName, funcTy)}; auto callOp{builder.create(loc, funcOp, mlirVecArgs)}; // If the result vector type is different from the original type, need @@ -2755,7 +2755,7 @@ void PPCIntrinsicLibrary::genMmaIntr(llvm::ArrayRef args) { auto context{builder.getContext()}; mlir::FunctionType intrFuncType{getMmaIrFuncType(context, IntrId)}; mlir::func::FuncOp funcOp{ - builder.addNamedFunction(loc, getMmaIrIntrName(IntrId), intrFuncType)}; + builder.createFunction(loc, getMmaIrIntrName(IntrId), intrFuncType)}; llvm::SmallVector intrArgs; // Depending on SubToFunc, change the subroutine call to a function call. @@ -2892,7 +2892,7 @@ void PPCIntrinsicLibrary::genVecStore(llvm::ArrayRef args) { auto funcType{ mlir::FunctionType::get(context, {stTy, addr.getType()}, std::nullopt)}; - mlir::func::FuncOp funcOp = builder.addNamedFunction(loc, fname, funcType); + mlir::func::FuncOp funcOp = builder.createFunction(loc, fname, funcType); llvm::SmallVector biArgs; diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp index 9bb10a42a399..dba2c30d1851 100644 --- a/flang/lib/Optimizer/Dialect/FIROps.cpp +++ b/flang/lib/Optimizer/Dialect/FIROps.cpp @@ -3677,10 +3677,19 @@ fir::parseSelector(mlir::OpAsmParser &parser, mlir::OperationState &result, return mlir::success(); } -mlir::func::FuncOp -fir::createFuncOp(mlir::Location loc, mlir::ModuleOp module, - llvm::StringRef name, mlir::FunctionType type, - llvm::ArrayRef attrs) { +mlir::func::FuncOp fir::createFuncOp(mlir::Location loc, mlir::ModuleOp module, + llvm::StringRef name, + mlir::FunctionType type, + llvm::ArrayRef attrs, + const mlir::SymbolTable *symbolTable) { + if (symbolTable) + if (auto f = symbolTable->lookup(name)) { +#ifdef EXPENSIVE_CHECKS + assert(f == module.lookupSymbol(name) && + "symbolTable and module out of sync"); +#endif + return f; + } if (auto f = module.lookupSymbol(name)) return f; mlir::OpBuilder modBuilder(module.getBodyRegion()); @@ -3692,7 +3701,16 @@ fir::createFuncOp(mlir::Location loc, mlir::ModuleOp module, fir::GlobalOp fir::createGlobalOp(mlir::Location loc, mlir::ModuleOp module, llvm::StringRef name, mlir::Type type, - llvm::ArrayRef attrs) { + llvm::ArrayRef attrs, + const mlir::SymbolTable *symbolTable) { + if (symbolTable) + if (auto g = symbolTable->lookup(name)) { +#ifdef EXPENSIVE_CHECKS + assert(g == module.lookupSymbol(name) && + "symbolTable and module out of sync"); +#endif + return g; + } if (auto g = module.lookupSymbol(name)) return g; mlir::OpBuilder modBuilder(module.getBodyRegion()); diff --git a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp index a11aa38c771b..f7820b6b8170 100644 --- a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp +++ b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp @@ -1004,10 +1004,8 @@ mlir::func::FuncOp SimplifyIntrinsicsPass::getOrCreateFunction( // We can also avoid this by using internal linkage, but // this may increase the size of final executable/shared library. std::string replacementName = mlir::Twine{baseName, "_simplified"}.str(); - mlir::ModuleOp module = builder.getModule(); // If we already have a function, just return it. - mlir::func::FuncOp newFunc = - fir::FirOpBuilder::getNamedFunction(module, replacementName); + mlir::func::FuncOp newFunc = builder.getNamedFunction(replacementName); mlir::FunctionType fType = typeGenerator(builder); if (newFunc) { assert(newFunc.getFunctionType() == fType && @@ -1017,8 +1015,7 @@ mlir::func::FuncOp SimplifyIntrinsicsPass::getOrCreateFunction( // Need to build the function! auto loc = mlir::UnknownLoc::get(builder.getContext()); - newFunc = - fir::FirOpBuilder::createFunction(loc, module, replacementName, fType); + newFunc = builder.createFunction(loc, replacementName, fType); auto inlineLinkage = mlir::LLVM::linkage::Linkage::LinkonceODR; auto linkage = mlir::LLVM::LinkageAttr::get(builder.getContext(), inlineLinkage); -- GitLab From eb08c0f1659d12524f58a01bf174177b8acedf2e Mon Sep 17 00:00:00 2001 From: Krystian Stasiowski Date: Tue, 2 Apr 2024 08:35:42 -0400 Subject: [PATCH 0188/1690] [Clang][Sema] Fix explicit specializations of member function templates with a deduced return type (#86817) Clang erroneously rejects the following: ``` template struct A { template auto f(); }; template<> template auto A::f(); // error: conflicting types for 'f' ``` This happens because the explicit specialization of `f` has its return type replaced with a dependent `AutoType` in `ActOnFunctionDeclarator`, but no such replacement occurs for the implicitly instantiated function template `A::f`. Since the return types don't match, the explicit specialization is diagnosed as an invalid redeclaration. This patch moves the replacement of the return type to `CheckFunctionDeclaration` so it also happens during instantiation. `setObjectOfFriendDecl` will have been called by then, so the `isFriend && CurContext->isDependentContext()` condition is made redundant & removed (as it already happens in `DeclContext::isDependentContext`). `Sema::IsOverload` only checks the _declared_ return type (which isn't changed by the adjustment), so adjusting the return type afterwards should be safe. --- .../clang-tidy/infrastructure/diagnostic.cpp | 4 +- clang/docs/ReleaseNotes.rst | 2 + clang/lib/Sema/SemaDecl.cpp | 46 ++++++++++++------- .../SemaCXX/deduced-return-type-cxx14.cpp | 18 ++++++++ 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/diagnostic.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/diagnostic.cpp index d0efc5ca7637..57d930b26e64 100644 --- a/clang-tools-extra/test/clang-tidy/infrastructure/diagnostic.cpp +++ b/clang-tools-extra/test/clang-tidy/infrastructure/diagnostic.cpp @@ -25,7 +25,7 @@ // RUN: not clang-tidy -checks='-*,modernize-use-override' %T/diagnostics/input.cpp -- -DCOMPILATION_ERROR 2>&1 | FileCheck -check-prefix=CHECK6 -implicit-check-not='{{warning:|error:}}' %s // RUN: clang-tidy -checks='-*,modernize-use-override,clang-diagnostic-macro-redefined' %s -- -DMACRO_FROM_COMMAND_LINE -std=c++20 | FileCheck -check-prefix=CHECK4 -implicit-check-not='{{warning:|error:}}' %s // RUN: clang-tidy -checks='-*,modernize-use-override,clang-diagnostic-macro-redefined,clang-diagnostic-literal-conversion' %s -- -DMACRO_FROM_COMMAND_LINE -std=c++20 -Wno-macro-redefined | FileCheck --check-prefix=CHECK7 -implicit-check-not='{{warning:|error:}}' %s -// RUN: not clang-tidy -checks='-*,modernize-use-override' %s -- -std=c++20 -DPR64602 | FileCheck -check-prefix=CHECK8 -implicit-check-not='{{warning:|error:}}' %s +// RUN: clang-tidy -checks='-*,modernize-use-override' %s -- -std=c++20 -DPR64602 // CHECK1: error: no input files [clang-diagnostic-error] // CHECK1: error: no such file or directory: '{{.*}}nonexistent.cpp' [clang-diagnostic-error] @@ -68,6 +68,4 @@ auto S<>::foo(auto) { return 1; } -// CHECK8: error: conflicting types for 'foo' [clang-diagnostic-error] -// CHECK8: note: previous declaration is here #endif diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index b2faab1f1525..3a84ff16a1e4 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -483,6 +483,8 @@ Bug Fixes to C++ Support following the first `::` were ignored). - Fix an out-of-bounds crash when checking the validity of template partial specializations. (part of #GH86757). - Fix an issue caused by not handling invalid cases when substituting into the parameter mapping of a constraint. Fixes (#GH86757). +- Fixed a bug that prevented member function templates of class templates declared with a deduced return type + from being explicitly specialized for a given implicit instantiation of the class template. Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 6ff85c0c5c29..5c1152896559 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -10124,23 +10124,6 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC, Diag(D.getDeclSpec().getVirtualSpecLoc(), diag::err_auto_fn_virtual); } - if (getLangOpts().CPlusPlus14 && - (NewFD->isDependentContext() || - (isFriend && CurContext->isDependentContext())) && - NewFD->getReturnType()->isUndeducedType()) { - // If the function template is referenced directly (for instance, as a - // member of the current instantiation), pretend it has a dependent type. - // This is not really justified by the standard, but is the only sane - // thing to do. - // FIXME: For a friend function, we have not marked the function as being - // a friend yet, so 'isDependentContext' on the FD doesn't work. - const FunctionProtoType *FPT = - NewFD->getType()->castAs(); - QualType Result = SubstAutoTypeDependent(FPT->getReturnType()); - NewFD->setType(Context.getFunctionType(Result, FPT->getParamTypes(), - FPT->getExtProtoInfo())); - } - // C++ [dcl.fct.spec]p3: // The inline specifier shall not appear on a block scope function // declaration. @@ -12112,6 +12095,35 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD, CheckConstPureAttributesUsage(*this, NewFD); + // C++ [dcl.spec.auto.general]p12: + // Return type deduction for a templated function with a placeholder in its + // declared type occurs when the definition is instantiated even if the + // function body contains a return statement with a non-type-dependent + // operand. + // + // C++ [temp.dep.expr]p3: + // An id-expression is type-dependent if it is a template-id that is not a + // concept-id and is dependent; or if its terminal name is: + // - [...] + // - associated by name lookup with one or more declarations of member + // functions of a class that is the current instantiation declared with a + // return type that contains a placeholder type, + // - [...] + // + // If this is a templated function with a placeholder in its return type, + // make the placeholder type dependent since it won't be deduced until the + // definition is instantiated. We do this here because it needs to happen + // for implicitly instantiated member functions/member function templates. + if (getLangOpts().CPlusPlus14 && + (NewFD->isDependentContext() && + NewFD->getReturnType()->isUndeducedType())) { + const FunctionProtoType *FPT = + NewFD->getType()->castAs(); + QualType NewReturnType = SubstAutoTypeDependent(FPT->getReturnType()); + NewFD->setType(Context.getFunctionType(NewReturnType, FPT->getParamTypes(), + FPT->getExtProtoInfo())); + } + // C++11 [dcl.constexpr]p8: // A constexpr specifier for a non-static member function that is not // a constructor declares that member function to be const. diff --git a/clang/test/SemaCXX/deduced-return-type-cxx14.cpp b/clang/test/SemaCXX/deduced-return-type-cxx14.cpp index 431d77ca785b..c33e07088ba3 100644 --- a/clang/test/SemaCXX/deduced-return-type-cxx14.cpp +++ b/clang/test/SemaCXX/deduced-return-type-cxx14.cpp @@ -237,6 +237,24 @@ namespace Templates { int (S::*(*p)())(double) = f; int (S::*(*q)())(double) = f; } + + template + struct MemberSpecialization { + auto f(); + template auto f(U); + template auto *f(U); + }; + + template<> + auto MemberSpecialization::f(); + + template<> + template + auto MemberSpecialization::f(U); + + template<> + template + auto *MemberSpecialization::f(U); } auto fwd_decl_using(); -- GitLab From 63d22f7a5b6afc515799f67c388bf5a8864274e4 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Tue, 2 Apr 2024 14:35:55 +0200 Subject: [PATCH 0189/1690] [MLIR][LLVM][SROA] Make GEP handling type agnostic (#86950) This commit removes SROA's type consistency constraints from LLVM dialect's GEPOp. The checks for valid indexing are now purely done by computing the GEP's offset with the aid of the data layout. To simplify handling of "nested subslots", we are tricking the SROA by handing in memory slots that hold byte array types. This ensures that subsequent accesses only need to check if their access will be in-bounds. This lifts the requirement of determining the sub-types for all but the first level of subslots. --- mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp | 241 +++++++++++++----- mlir/test/Dialect/LLVMIR/sroa.mlir | 130 ++++++++++ 2 files changed, 312 insertions(+), 59 deletions(-) diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp index f171bf7cc4be..06c1fdd2eb2d 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp @@ -20,6 +20,8 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/TypeSwitch.h" +#define DEBUG_TYPE "sroa" + using namespace mlir; //===----------------------------------------------------------------------===// @@ -431,10 +433,147 @@ DeletionKind LLVM::GEPOp::removeBlockingUses( return DeletionKind::Delete; } -static bool isFirstIndexZero(LLVM::GEPOp gep) { - IntegerAttr index = - llvm::dyn_cast_if_present(gep.getIndices()[0]); - return index && index.getInt() == 0; +/// Returns the amount of bytes the provided GEP elements will offset the +/// pointer by. Returns nullopt if no constant offset could be computed. +static std::optional gepToByteOffset(const DataLayout &dataLayout, + LLVM::GEPOp gep) { + // Collects all indices. + SmallVector indices; + for (auto index : gep.getIndices()) { + auto constIndex = dyn_cast(index); + if (!constIndex) + return {}; + int64_t gepIndex = constIndex.getInt(); + // Negative indices are not supported. + if (gepIndex < 0) + return {}; + indices.push_back(gepIndex); + } + + Type currentType = gep.getElemType(); + uint64_t offset = indices[0] * dataLayout.getTypeSize(currentType); + + for (uint64_t index : llvm::drop_begin(indices)) { + bool shouldCancel = + TypeSwitch(currentType) + .Case([&](LLVM::LLVMArrayType arrayType) { + offset += + index * dataLayout.getTypeSize(arrayType.getElementType()); + currentType = arrayType.getElementType(); + return false; + }) + .Case([&](LLVM::LLVMStructType structType) { + ArrayRef body = structType.getBody(); + assert(index < body.size() && "expected valid struct indexing"); + for (uint32_t i : llvm::seq(index)) { + if (!structType.isPacked()) + offset = llvm::alignTo( + offset, dataLayout.getTypeABIAlignment(body[i])); + offset += dataLayout.getTypeSize(body[i]); + } + + // Align for the current type as well. + if (!structType.isPacked()) + offset = llvm::alignTo( + offset, dataLayout.getTypeABIAlignment(body[index])); + currentType = body[index]; + return false; + }) + .Default([&](Type type) { + LLVM_DEBUG(llvm::dbgs() + << "[sroa] Unsupported type for offset computations" + << type << "\n"); + return true; + }); + + if (shouldCancel) + return std::nullopt; + } + + return offset; +} + +namespace { +/// A struct that stores both the index into the aggregate type of the slot as +/// well as the corresponding byte offset in memory. +struct SubslotAccessInfo { + /// The parent slot's index that the access falls into. + uint32_t index; + /// The offset into the subslot of the access. + uint64_t subslotOffset; +}; +} // namespace + +/// Computes subslot access information for an access into `slot` with the given +/// offset. +/// Returns nullopt when the offset is out-of-bounds or when the access is into +/// the padding of `slot`. +static std::optional +getSubslotAccessInfo(const DestructurableMemorySlot &slot, + const DataLayout &dataLayout, LLVM::GEPOp gep) { + std::optional offset = gepToByteOffset(dataLayout, gep); + if (!offset) + return {}; + + // Helper to check that a constant index is in the bounds of the GEP index + // representation. LLVM dialects's GEP arguments have a limited bitwidth, thus + // this additional check is necessary. + auto isOutOfBoundsGEPIndex = [](uint64_t index) { + return index >= (1 << LLVM::kGEPConstantBitWidth); + }; + + Type type = slot.elemType; + if (*offset >= dataLayout.getTypeSize(type)) + return {}; + return TypeSwitch>(type) + .Case([&](LLVM::LLVMArrayType arrayType) + -> std::optional { + // Find which element of the array contains the offset. + uint64_t elemSize = dataLayout.getTypeSize(arrayType.getElementType()); + uint64_t index = *offset / elemSize; + if (isOutOfBoundsGEPIndex(index)) + return {}; + return SubslotAccessInfo{static_cast(index), + *offset - (index * elemSize)}; + }) + .Case([&](LLVM::LLVMStructType structType) + -> std::optional { + uint64_t distanceToStart = 0; + // Walk over the elements of the struct to find in which of + // them the offset is. + for (auto [index, elem] : llvm::enumerate(structType.getBody())) { + uint64_t elemSize = dataLayout.getTypeSize(elem); + if (!structType.isPacked()) { + distanceToStart = llvm::alignTo( + distanceToStart, dataLayout.getTypeABIAlignment(elem)); + // If the offset is in padding, cancel the rewrite. + if (offset < distanceToStart) + return {}; + } + + if (offset < distanceToStart + elemSize) { + if (isOutOfBoundsGEPIndex(index)) + return {}; + // The offset is within this element, stop iterating the + // struct and return the index. + return SubslotAccessInfo{static_cast(index), + *offset - distanceToStart}; + } + + // The offset is not within this element, continue walking + // over the struct. + distanceToStart += elemSize; + } + + return {}; + }); +} + +/// Constructs a byte array type of the given size. +static LLVM::LLVMArrayType getByteArrayType(MLIRContext *context, + unsigned size) { + auto byteType = IntegerType::get(context, 8); + return LLVM::LLVMArrayType::get(context, byteType, size); } LogicalResult LLVM::GEPOp::ensureOnlySafeAccesses( @@ -442,18 +581,17 @@ LogicalResult LLVM::GEPOp::ensureOnlySafeAccesses( const DataLayout &dataLayout) { if (getBase() != slot.ptr) return success(); - if (slot.elemType != getElemType()) - return failure(); - if (!isFirstIndexZero(*this)) + std::optional gepOffset = gepToByteOffset(dataLayout, *this); + if (!gepOffset) return failure(); - // Dynamic indices can be out-of-bounds (even negative), so an access with - // dynamic indices can never be considered safe. - if (!getDynamicIndices().empty()) + uint64_t slotSize = dataLayout.getTypeSize(slot.elemType); + // Check that the access is strictly inside the slot. + if (*gepOffset >= slotSize) return failure(); - Type reachedType = getResultPtrElementType(); - if (!reachedType) - return failure(); - mustBeSafelyUsed.emplace_back({getResult(), reachedType}); + // Every access that remains in bounds of the remaining slot is considered + // legal. + mustBeSafelyUsed.emplace_back( + {getRes(), getByteArrayType(getContext(), slotSize - *gepOffset)}); return success(); } @@ -464,23 +602,25 @@ bool LLVM::GEPOp::canRewire(const DestructurableMemorySlot &slot, if (!isa(getBase().getType())) return false; - if (getBase() != slot.ptr || slot.elemType != getElemType()) - return false; - if (!isFirstIndexZero(*this)) - return false; - // Dynamic indices can be out-of-bounds (even negative), so an access with - // dynamic indices can never be properly rewired. - if (!getDynamicIndices().empty()) - return false; - Type reachedType = getResultPtrElementType(); - if (!reachedType || getIndices().size() < 2) + if (getBase() != slot.ptr) return false; - auto firstLevelIndex = dyn_cast(getIndices()[1]); - if (!firstLevelIndex) + std::optional accessInfo = + getSubslotAccessInfo(slot, dataLayout, *this); + if (!accessInfo) return false; - mustBeSafelyUsed.emplace_back({getResult(), reachedType}); - assert(slot.elementPtrs.contains(firstLevelIndex)); - usedIndices.insert(firstLevelIndex); + auto indexAttr = + IntegerAttr::get(IntegerType::get(getContext(), 32), accessInfo->index); + assert(slot.elementPtrs.contains(indexAttr)); + usedIndices.insert(indexAttr); + + // The remainder of the subslot should be accesses in-bounds. Thus, we create + // a dummy slot with the size of the remainder. + Type subslotType = slot.elementPtrs.lookup(indexAttr); + uint64_t slotSize = dataLayout.getTypeSize(subslotType); + LLVM::LLVMArrayType remainingSlotType = + getByteArrayType(getContext(), slotSize - accessInfo->subslotOffset); + mustBeSafelyUsed.emplace_back({getRes(), remainingSlotType}); + return true; } @@ -488,36 +628,19 @@ DeletionKind LLVM::GEPOp::rewire(const DestructurableMemorySlot &slot, DenseMap &subslots, RewriterBase &rewriter, const DataLayout &dataLayout) { - IntegerAttr firstLevelIndex = - llvm::dyn_cast_if_present(getIndices()[1]); - const MemorySlot &newSlot = subslots.at(firstLevelIndex); - - ArrayRef remainingIndices = getRawConstantIndices().slice(2); - - // If the GEP would become trivial after this transformation, eliminate it. - // A GEP should only be eliminated if it has no indices (except the first - // pointer index), as simplifying GEPs with all-zero indices would eliminate - // structure information useful for further destruction. - if (remainingIndices.empty()) { - rewriter.replaceAllUsesWith(getResult(), newSlot.ptr); - return DeletionKind::Delete; - } - - rewriter.modifyOpInPlace(*this, [&]() { - // Rewire the indices by popping off the second index. - // Start with a single zero, then add the indices beyond the second. - SmallVector newIndices(1); - newIndices.append(remainingIndices.begin(), remainingIndices.end()); - setRawConstantIndices(newIndices); - - // Rewire the pointed type. - setElemType(newSlot.elemType); - - // Rewire the pointer. - getBaseMutable().assign(newSlot.ptr); - }); - - return DeletionKind::Keep; + std::optional accessInfo = + getSubslotAccessInfo(slot, dataLayout, *this); + assert(accessInfo && "expected access info to be checked before"); + auto indexAttr = + IntegerAttr::get(IntegerType::get(getContext(), 32), accessInfo->index); + const MemorySlot &newSlot = subslots.at(indexAttr); + + auto byteType = IntegerType::get(rewriter.getContext(), 8); + auto newPtr = rewriter.createOrFold( + getLoc(), getResult().getType(), byteType, newSlot.ptr, + ArrayRef(accessInfo->subslotOffset), getInbounds()); + rewriter.replaceAllUsesWith(getResult(), newPtr); + return DeletionKind::Delete; } //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/LLVMIR/sroa.mlir b/mlir/test/Dialect/LLVMIR/sroa.mlir index 3f4d17c6a43f..fe1531d988a4 100644 --- a/mlir/test/Dialect/LLVMIR/sroa.mlir +++ b/mlir/test/Dialect/LLVMIR/sroa.mlir @@ -82,6 +82,27 @@ llvm.func @multi_level_indirect() -> i32 { // ----- +// This verifies that a nested GEP's users are checked properly. In this case +// the load goes over the bounds of the memory slot and thus should block the +// splitting of the alloca. + +// CHECK-LABEL: llvm.func @nested_access_over_slot_bound +llvm.func @nested_access_over_slot_bound() -> i64 { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<(i32, struct<( + %1 = llvm.alloca %0 x !llvm.struct<(i32, struct<(array<10 x i32>)>, i32)> {alignment = 8 : i64} : (i32) -> !llvm.ptr + // CHECK: %[[GEP0:.*]] = llvm.getelementptr inbounds %[[ALLOCA]] + %2 = llvm.getelementptr inbounds %1[0, 1, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i32, struct<(array<10 x i32>)>, i32)> + // CHECK: %[[GEP1:.*]] = llvm.getelementptr inbounds %[[GEP0]] + %3 = llvm.getelementptr inbounds %2[0, 9] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<10 x i32> + // CHECK: %[[RES:.*]] = llvm.load %[[GEP1]] + %4 = llvm.load %3 : !llvm.ptr -> i64 + // CHECK: llvm.return %[[RES]] : i64 + llvm.return %4 : i64 +} + +// ----- + // CHECK-LABEL: llvm.func @resolve_alias // CHECK-SAME: (%[[ARG:.*]]: i32) llvm.func @resolve_alias(%arg: i32) -> i32 { @@ -318,3 +339,112 @@ llvm.func @store_to_memory(%arg: !llvm.ptr) { llvm.store %1, %arg : !llvm.ptr, !llvm.ptr llvm.return } + +// ----- + +// CHECK-LABEL: llvm.func @type_mismatch_array_access +// CHECK-SAME: %[[ARG:.*]]: i32 +llvm.func @type_mismatch_array_access(%arg: i32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<(i32, i32, i32)> : (i32) -> !llvm.ptr + %2 = llvm.getelementptr %1[8] : (!llvm.ptr) -> !llvm.ptr, i8 + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] + llvm.store %arg, %2 : i32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @type_mismatch_struct_access +// CHECK-SAME: %[[ARG:.*]]: i32 +llvm.func @type_mismatch_struct_access(%arg: i32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<(i32, i32, i32)> : (i32) -> !llvm.ptr + %2 = llvm.getelementptr %1[0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i32, i32)> + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] + llvm.store %arg, %2 : i32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @index_in_final_padding +llvm.func @index_in_final_padding(%arg: i32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i8)> + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i8)> : (i32) -> !llvm.ptr + // CHECK: = llvm.getelementptr %[[ALLOCA]][7] : (!llvm.ptr) -> !llvm.ptr, i8 + %2 = llvm.getelementptr %1[7] : (!llvm.ptr) -> !llvm.ptr, i8 + llvm.store %arg, %2 : i32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @index_out_of_bounds +llvm.func @index_out_of_bounds(%arg: i32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32)> + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32)> : (i32) -> !llvm.ptr + // CHECK: = llvm.getelementptr %[[ALLOCA]][9] : (!llvm.ptr) -> !llvm.ptr, i8 + %2 = llvm.getelementptr %1[9] : (!llvm.ptr) -> !llvm.ptr, i8 + llvm.store %arg, %2 : i32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @index_in_padding +llvm.func @index_in_padding(%arg: i16) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, i32)> + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i16, i32)> : (i32) -> !llvm.ptr + // CHECK: = llvm.getelementptr %[[ALLOCA]][2] : (!llvm.ptr) -> !llvm.ptr, i8 + %2 = llvm.getelementptr %1[2] : (!llvm.ptr) -> !llvm.ptr, i8 + llvm.store %arg, %2 : i16, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @index_not_in_padding_because_packed +// CHECK-SAME: %[[ARG:.*]]: i16 +llvm.func @index_not_in_padding_because_packed(%arg: i16) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", packed (i16, i32)> : (i32) -> !llvm.ptr + %2 = llvm.getelementptr %1[2] : (!llvm.ptr) -> !llvm.ptr, i8 + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] + llvm.store %arg, %2 : i16, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @no_crash_on_negative_gep_index +// CHECK-SAME: %[[ARG:.*]]: f16 +llvm.func @no_crash_on_negative_gep_index(%arg: f16) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK: llvm.getelementptr %[[ALLOCA]][-1] : (!llvm.ptr) -> !llvm.ptr, f32 + %2 = llvm.getelementptr %1[-1] : (!llvm.ptr) -> !llvm.ptr, f32 + llvm.store %arg, %2 : f16, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @out_of_bound_gep_array_access +// CHECK-SAME: %[[ARG:.*]]: i32 +llvm.func @out_of_bound_gep_array_access(%arg: i32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32)> : (i32) -> !llvm.ptr + %2 = llvm.getelementptr %1[0, 4] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<4 x i8> + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] + llvm.store %arg, %2 : i32, !llvm.ptr + llvm.return +} -- GitLab From 56aeac47ab0858db9f447b5ec43b660d9035167f Mon Sep 17 00:00:00 2001 From: Mitch Phillips Date: Tue, 2 Apr 2024 14:44:11 +0200 Subject: [PATCH 0190/1690] Revert "[mlir] Reland the dialect conversion hanging use fix (#87297)" This reverts commit 49a4ec20a8be5888cbf225bab340dbaf204902c7. Reason: Broke the ASan build bot with a memory leak. See the comments at https://github.com/llvm/llvm-project/pull/87297 for more information. --- mlir/lib/Transforms/Utils/DialectConversion.cpp | 2 -- .../TosaToLinalg/tosa-to-linalg-invalid.mlir | 13 ------------- 2 files changed, 15 deletions(-) diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index 270ac0a08689..8671c1008902 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -279,8 +279,6 @@ public: auto &blockOps = block->getOperations(); while (!blockOps.empty()) blockOps.remove(blockOps.begin()); - for (auto arg : block->getArguments()) - arg.dropAllUses(); block->dropAllUses(); if (block->getParent()) block->erase(); diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir index 6494e1b27194..17eec5936918 100644 --- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir +++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir @@ -15,16 +15,3 @@ func.func @tensor_with_unknown_rank(%arg0: tensor<*xi8>) -> tensor<*xi8> { %0 = "tosa.abs"(%arg0) : (tensor<*xi8>) -> tensor<*xi8> return %0 : tensor<*xi8> } - -// ----- - -// CHECK-LABEL: @unranked_add -func.func @unranked_add(%arg0 : tensor<10x10xf32> , %arg1 : tensor<10x10xf32>, %arg2 : tensor<*xf32>) -> (tensor<10x10xf32>) { - // expected-error@+3 {{failed to legalize operation 'tosa.add'}} - %reduce = tosa.reduce_max %arg0 {axis = 1 : i32} : (tensor<10x10xf32>) -> tensor<10x1xf32> - %1 = tosa.add %reduce, %arg1 : (tensor<10x1xf32>, tensor<10x10xf32>) -> tensor<10x10xf32> - %0 = tosa.add %1, %arg2 : (tensor<10x10xf32>, tensor<*xf32>) -> tensor<*xf32> - %2 = tosa.reshape %0 {new_shape = array} : (tensor<*xf32>) -> tensor<10x10xf32> - return %2 : tensor<10x10xf32> -} - -- GitLab From cb9cf331fa69bb8319886cdca043a742b4d19ce5 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 2 Apr 2024 05:52:27 -0700 Subject: [PATCH 0191/1690] [SLP][NFC]Do not lookup in MinBWs, reuse previously used iterator. --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 1ffc39a90674..b1940e3feed9 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -12297,7 +12297,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { Value *V = Builder.CreateBinOp( static_cast(E->getOpcode()), LHS, RHS); - propagateIRFlags(V, E->Scalars, VL0, !MinBWs.contains(E)); + propagateIRFlags(V, E->Scalars, VL0, It == MinBWs.end()); if (auto *I = dyn_cast(V)) V = propagateMetadata(I, E->Scalars); @@ -12598,8 +12598,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { }, Mask, &OpScalars, &AltScalars); - propagateIRFlags(V0, OpScalars, E->getMainOp(), !MinBWs.contains(E)); - propagateIRFlags(V1, AltScalars, E->getAltOp(), !MinBWs.contains(E)); + propagateIRFlags(V0, OpScalars, E->getMainOp(), It == MinBWs.end()); + propagateIRFlags(V1, AltScalars, E->getAltOp(), It == MinBWs.end()); Value *V = Builder.CreateShuffleVector(V0, V1, Mask); if (auto *I = dyn_cast(V)) { -- GitLab From 0b9528d6bd0bfde5702b1ee5ed8a249d354434f1 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Tue, 2 Apr 2024 15:18:38 +0200 Subject: [PATCH 0192/1690] [clang] CTAD: Track template template type parameters that referenced in the template arguments of the RHS. (#85405) Fixes https://github.com/llvm/llvm-project/issues/85385. The Finder was missing for this case, for the crash test, the template parameter TTP was incorrectly considered as not referenced/appeared in the template arguments of the right hand side of the alias template decl, thus the synthesized deduction decl doesn't contain this TTP in the template parameter list, but we have references in the declaration, thus it caused crashes. --- clang/lib/Sema/SemaTemplate.cpp | 6 ++++++ clang/test/SemaCXX/cxx20-ctad-type-alias.cpp | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index befec401c8ee..a2b8cc14ca76 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -2720,6 +2720,12 @@ SmallVector TemplateParamsReferencedInTemplateArgumentList( return true; } + bool TraverseTemplateName(TemplateName Template) { + if (auto *TD = Template.getAsTemplateDecl()) + MarkAppeared(TD); + return RecursiveASTVisitor::TraverseTemplateName(Template); + } + void MarkAppeared(NamedDecl *ND) { if (TemplateParams.contains(ND)) ReferencedTemplateParams.insert(ND); diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp index ce403285b0f5..b71cd46f884d 100644 --- a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp +++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp @@ -259,3 +259,23 @@ using Bar2 = Foo; // expected-error {{extraneous template parameter list in a Bar2 b = 1; // expected-error {{no viable constructor or deduction guide for deduction of template arguments}} } // namespace test19 + +// GH85385 +namespace test20 { +template