diff options
Diffstat (limited to 'llvm/test/CodeGen')
86 files changed, 19392 insertions, 6572 deletions
diff --git a/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes.mir b/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes.mir deleted file mode 100644 index 09eb18b..0000000 --- a/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes.mir +++ /dev/null @@ -1,364 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5 -# RUN: llc -run-pass=machine-combiner -mcpu=neoverse-n2 -mtriple=aarch64-none-linux-gnu -verify-machineinstrs %s -o - | FileCheck %s - ---- -name: split_loads_to_fpr128 -body: | - bb.0.entry: - liveins: $x0, $x1, $x2, $x3, $x4 - - ; CHECK-LABEL: name: split_loads_to_fpr128 - ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0 - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2 - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3 - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4 - ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], killed [[COPY1]], 0, 1 - ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i32]], %subreg.ssub - ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, killed [[COPY2]] - ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr32 = LDRSui [[COPY3]], 0 - ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.ssub - ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i32 [[SECOND_REG]], 1, killed [[COPY4]] - ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_1]], [[LD1_1]] - ; CHECK-NEXT: $q0 = COPY [[ZIP]] - ; CHECK-NEXT: RET_ReallyLR implicit $q0 - %0:gpr64common = COPY $x0 - %1:gpr64common = COPY $x1 - %2:gpr64common = COPY $x2 - %3:gpr64common = COPY $x3 - %4:gpr64common = COPY $x4 - %5:fpr32 = LDRSroX %0, killed %1, 0, 1 - %6:fpr128 = SUBREG_TO_REG 0, killed %5, %subreg.ssub - %7:fpr128 = LD1i32 %6, 1, killed %2 - %8:fpr128 = LD1i32 %7, 2, killed %3 - %9:fpr128 = LD1i32 %8, 3, killed %4 - $q0 = COPY %9 - RET_ReallyLR implicit $q0 - ---- -name: split_loads_to_fpr128_ui -body: | - bb.0.entry: - liveins: $x0, $x1, $x2, $x3, $x4 - - ; CHECK-LABEL: name: split_loads_to_fpr128_ui - ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0 - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2 - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3 - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4 - ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSui [[COPY]], 0 - ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i32]], %subreg.ssub - ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, killed [[COPY1]] - ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr32 = LDRSui [[COPY2]], 0 - ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.ssub - ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i32 [[SECOND_REG]], 1, killed [[COPY3]] - ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_1]], [[LD1_1]] - ; CHECK-NEXT: $q0 = COPY [[ZIP]] - ; CHECK-NEXT: RET_ReallyLR implicit $q0 - %0:gpr64common = COPY $x0 - %1:gpr64common = COPY $x1 - %2:gpr64common = COPY $x2 - %3:gpr64common = COPY $x3 - %4:gpr64common = COPY $x4 - %5:fpr32 = LDRSui %0, 0 - %6:fpr128 = SUBREG_TO_REG 0, killed %5, %subreg.ssub - %7:fpr128 = LD1i32 %6, 1, killed %1 - %8:fpr128 = LD1i32 %7, 2, killed %2 - %9:fpr128 = LD1i32 %8, 3, killed %3 - $q0 = COPY %9 - RET_ReallyLR implicit $q0 - ---- -name: split_loads_to_fpr128_i16 -body: | - bb.0.entry: - liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8 - - ; CHECK-LABEL: name: split_loads_to_fpr128_i16 - ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0 - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2 - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3 - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4 - ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64common = COPY $x5 - ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr64common = COPY $x6 - ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr64common = COPY $x7 - ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr64common = COPY $x8 - ; CHECK-NEXT: [[LD_i16:%[0-9]+]]:fpr16 = LDRHroX [[COPY]], killed [[COPY1]], 0, 1 - ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i16]], %subreg.hsub - ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i16 [[FIRST_REG]], 1, killed [[COPY2]] - ; CHECK-NEXT: [[LD0_2:%[0-9]+]]:fpr128 = LD1i16 [[LD0_1]], 2, killed [[COPY3]] - ; CHECK-NEXT: [[LD0_3:%[0-9]+]]:fpr128 = LD1i16 [[LD0_2]], 3, killed [[COPY4]] - ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr16 = LDRHui [[COPY5]], 0 - ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.hsub - ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i16 [[SECOND_REG]], 1, killed [[COPY6]] - ; CHECK-NEXT: [[LD1_2:%[0-9]+]]:fpr128 = LD1i16 [[LD1_1]], 2, killed [[COPY7]] - ; CHECK-NEXT: [[LD1_3:%[0-9]+]]:fpr128 = LD1i16 [[LD1_2]], 3, killed [[COPY8]] - ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_3]], [[LD1_3]] - ; CHECK-NEXT: $q0 = COPY [[ZIP]] - ; CHECK-NEXT: RET_ReallyLR implicit $q0 - %0:gpr64common = COPY $x0 - %1:gpr64common = COPY $x1 - %2:gpr64common = COPY $x2 - %3:gpr64common = COPY $x3 - %4:gpr64common = COPY $x4 - %5:gpr64common = COPY $x5 - %6:gpr64common = COPY $x6 - %7:gpr64common = COPY $x7 - %8:gpr64common = COPY $x8 - %9:fpr16 = LDRHroX %0, killed %1, 0, 1 - %10:fpr128 = SUBREG_TO_REG 0, killed %9, %subreg.hsub - %11:fpr128 = LD1i16 %10, 1, killed %2 - %12:fpr128 = LD1i16 %11, 2, killed %3 - %13:fpr128 = LD1i16 %12, 3, killed %4 - %14:fpr128 = LD1i16 %13, 4, killed %5 - %15:fpr128 = LD1i16 %14, 5, killed %6 - %16:fpr128 = LD1i16 %15, 6, killed %7 - %17:fpr128 = LD1i16 %16, 7, killed %8 - $q0 = COPY %17 - RET_ReallyLR implicit $q0 - ---- -name: split_loads_to_fpr128_i16_ui -body: | - bb.0.entry: - liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8 - - ; CHECK-LABEL: name: split_loads_to_fpr128_i16_ui - ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0 - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2 - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3 - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4 - ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64common = COPY $x5 - ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr64common = COPY $x6 - ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr64common = COPY $x7 - ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr64common = COPY $x8 - ; CHECK-NEXT: [[LD_i16:%[0-9]+]]:fpr16 = LDRHui [[COPY]], 0 - ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i16]], %subreg.hsub - ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i16 [[FIRST_REG]], 1, killed [[COPY1]] - ; CHECK-NEXT: [[LD0_2:%[0-9]+]]:fpr128 = LD1i16 [[LD0_1]], 2, killed [[COPY2]] - ; CHECK-NEXT: [[LD0_3:%[0-9]+]]:fpr128 = LD1i16 [[LD0_2]], 3, killed [[COPY3]] - ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr16 = LDRHui [[COPY4]], 0 - ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.hsub - ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i16 [[SECOND_REG]], 1, killed [[COPY5]] - ; CHECK-NEXT: [[LD1_2:%[0-9]+]]:fpr128 = LD1i16 [[LD1_1]], 2, killed [[COPY6]] - ; CHECK-NEXT: [[LD1_3:%[0-9]+]]:fpr128 = LD1i16 [[LD1_2]], 3, killed [[COPY7]] - ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_3]], [[LD1_3]] - ; CHECK-NEXT: $q0 = COPY [[ZIP]] - ; CHECK-NEXT: RET_ReallyLR implicit $q0 - %0:gpr64common = COPY $x0 - %1:gpr64common = COPY $x1 - %2:gpr64common = COPY $x2 - %3:gpr64common = COPY $x3 - %4:gpr64common = COPY $x4 - %5:gpr64common = COPY $x5 - %6:gpr64common = COPY $x6 - %7:gpr64common = COPY $x7 - %8:gpr64common = COPY $x8 - %9:fpr16 = LDRHui %0, 0 - %10:fpr128 = SUBREG_TO_REG 0, killed %9, %subreg.hsub - %11:fpr128 = LD1i16 %10, 1, killed %1 - %12:fpr128 = LD1i16 %11, 2, killed %2 - %13:fpr128 = LD1i16 %12, 3, killed %3 - %14:fpr128 = LD1i16 %13, 4, killed %4 - %15:fpr128 = LD1i16 %14, 5, killed %5 - %16:fpr128 = LD1i16 %15, 6, killed %6 - %17:fpr128 = LD1i16 %16, 7, killed %7 - $q0 = COPY %17 - RET_ReallyLR implicit $q0 - ---- -name: split_loads_to_fpr128_i8 -body: | - bb.0.entry: - liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16 - - ; CHECK-LABEL: name: split_loads_to_fpr128_i8 - ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0 - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2 - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3 - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4 - ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64common = COPY $x5 - ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr64common = COPY $x6 - ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr64common = COPY $x7 - ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr64common = COPY $x8 - ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gpr64common = COPY $x9 - ; CHECK-NEXT: [[COPY10:%[0-9]+]]:gpr64common = COPY $x10 - ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gpr64common = COPY $x11 - ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gpr64common = COPY $x12 - ; CHECK-NEXT: [[COPY13:%[0-9]+]]:gpr64common = COPY $x13 - ; CHECK-NEXT: [[COPY14:%[0-9]+]]:gpr64common = COPY $x14 - ; CHECK-NEXT: [[COPY15:%[0-9]+]]:gpr64common = COPY $x15 - ; CHECK-NEXT: [[COPY16:%[0-9]+]]:gpr64common = COPY $x16 - ; CHECK-NEXT: [[LD_i8:%[0-9]+]]:fpr8 = LDRBroX [[COPY]], killed [[COPY1]], 0, 0 - ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i8]], %subreg.bsub - ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i8 [[FIRST_REG]], 1, killed [[COPY2]] - ; CHECK-NEXT: [[LD0_2:%[0-9]+]]:fpr128 = LD1i8 [[LD0_1]], 2, killed [[COPY3]] - ; CHECK-NEXT: [[LD0_3:%[0-9]+]]:fpr128 = LD1i8 [[LD0_2]], 3, killed [[COPY4]] - ; CHECK-NEXT: [[LD0_4:%[0-9]+]]:fpr128 = LD1i8 [[LD0_3]], 4, killed [[COPY5]] - ; CHECK-NEXT: [[LD0_5:%[0-9]+]]:fpr128 = LD1i8 [[LD0_4]], 5, killed [[COPY6]] - ; CHECK-NEXT: [[LD0_6:%[0-9]+]]:fpr128 = LD1i8 [[LD0_5]], 6, killed [[COPY7]] - ; CHECK-NEXT: [[LD0_7:%[0-9]+]]:fpr128 = LD1i8 [[LD0_6]], 7, killed [[COPY8]] - ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr8 = LDRBui [[COPY9]], 0 - ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.bsub - ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i8 [[SECOND_REG]], 1, killed [[COPY10]] - ; CHECK-NEXT: [[LD1_2:%[0-9]+]]:fpr128 = LD1i8 [[LD1_1]], 2, killed [[COPY11]] - ; CHECK-NEXT: [[LD1_3:%[0-9]+]]:fpr128 = LD1i8 [[LD1_2]], 3, killed [[COPY12]] - ; CHECK-NEXT: [[LD1_4:%[0-9]+]]:fpr128 = LD1i8 [[LD1_3]], 4, killed [[COPY13]] - ; CHECK-NEXT: [[LD1_5:%[0-9]+]]:fpr128 = LD1i8 [[LD1_4]], 5, killed [[COPY14]] - ; CHECK-NEXT: [[LD1_6:%[0-9]+]]:fpr128 = LD1i8 [[LD1_5]], 6, killed [[COPY15]] - ; CHECK-NEXT: [[LD1_7:%[0-9]+]]:fpr128 = LD1i8 [[LD1_6]], 7, killed [[COPY16]] - ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_7]], [[LD1_7]] - ; CHECK-NEXT: $q0 = COPY [[ZIP]] - ; CHECK-NEXT: RET_ReallyLR implicit $q0 - %0:gpr64common = COPY $x0 - %1:gpr64common = COPY $x1 - %2:gpr64common = COPY $x2 - %3:gpr64common = COPY $x3 - %4:gpr64common = COPY $x4 - %5:gpr64common = COPY $x5 - %6:gpr64common = COPY $x6 - %7:gpr64common = COPY $x7 - %8:gpr64common = COPY $x8 - %9:gpr64common = COPY $x9 - %10:gpr64common = COPY $x10 - %11:gpr64common = COPY $x11 - %12:gpr64common = COPY $x12 - %13:gpr64common = COPY $x13 - %14:gpr64common = COPY $x14 - %15:gpr64common = COPY $x15 - %16:gpr64common = COPY $x16 - %17:fpr8 = LDRBroX %0, killed %1, 0, 0 - %18:fpr128 = SUBREG_TO_REG 0, killed %17, %subreg.bsub - %19:fpr128 = LD1i8 %18, 1, killed %2 - %20:fpr128 = LD1i8 %19, 2, killed %3 - %21:fpr128 = LD1i8 %20, 3, killed %4 - %22:fpr128 = LD1i8 %21, 4, killed %5 - %23:fpr128 = LD1i8 %22, 5, killed %6 - %24:fpr128 = LD1i8 %23, 6, killed %7 - %25:fpr128 = LD1i8 %24, 7, killed %8 - %26:fpr128 = LD1i8 %25, 8, killed %9 - %27:fpr128 = LD1i8 %26, 9, killed %10 - %28:fpr128 = LD1i8 %27, 10, killed %11 - %29:fpr128 = LD1i8 %28, 11, killed %12 - %30:fpr128 = LD1i8 %29, 12, killed %13 - %31:fpr128 = LD1i8 %30, 13, killed %14 - %32:fpr128 = LD1i8 %31, 14, killed %15 - %33:fpr128 = LD1i8 %32, 15, killed %16 - $q0 = COPY %33 - RET_ReallyLR implicit $q0 - ---- -name: negative_pattern_missing_lanes -body: | - bb.0.entry: - liveins: $x0, $x1 - - ; CHECK-LABEL: name: negative_pattern_missing_lanes - ; CHECK: [[LD1:%.*]]:fpr128 = LDRQui $x1, 0 - ; CHECK-NEXT: [[LD2:%.*]]:fpr128 = LD1i32 [[LD1]] - - %0:gpr64common = COPY $x0 - %1:fpr128 = LDRQui $x1, 0 - %2:fpr128 = LD1i32 %1, 3, %0 - $q0 = COPY %2 - RET_ReallyLR implicit $q0 - ---- -name: out_of_order_lanes -body: | - bb.0.entry: - liveins: $x0, $x1, $x2, $x3, $x4 - - ; CHECK-LABEL: name: out_of_order_lanes - ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0 - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2 - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3 - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4 - ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], killed [[COPY1]], 0, 1 - ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i32]], %subreg.ssub - ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, killed [[COPY3]] - ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr32 = LDRSui [[COPY2]], 0 - ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.ssub - ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i32 [[SECOND_REG]], 1, killed [[COPY4]] - ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_1]], [[LD1_1]] - ; CHECK-NEXT: $q0 = COPY [[ZIP]] - ; CHECK-NEXT: RET_ReallyLR implicit $q0 - %0:gpr64common = COPY $x0 - %1:gpr64common = COPY $x1 - %2:gpr64common = COPY $x2 - %3:gpr64common = COPY $x3 - %4:gpr64common = COPY $x4 - %5:fpr32 = LDRSroX %0, killed %1, 0, 1 - %6:fpr128 = SUBREG_TO_REG 0, killed %5, %subreg.ssub - %7:fpr128 = LD1i32 %6, 2, killed %2 - %8:fpr128 = LD1i32 %7, 1, killed %3 - %9:fpr128 = LD1i32 %8, 3, killed %4 - $q0 = COPY %9 - RET_ReallyLR implicit $q0 - ---- -name: negative_pattern_no_subreg_to_reg -body: | - bb.0.entry: - liveins: $x0, $x1, $x2, $x3 - - ; CHECK-LABEL: name: negative_pattern_no_subreg_to_reg - ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0 - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2 - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3 - ; CHECK-NEXT: [[INITIAL_VEC:%[0-9]+]]:fpr128 = LDRQui [[COPY]], 0 - ; CHECK-NEXT: [[LD_LANE_1:%[0-9]+]]:fpr128 = LD1i32 [[INITIAL_VEC]], 1, killed [[COPY1]] - ; CHECK-NEXT: [[LD_LANE_2:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_1]], 2, killed [[COPY2]] - ; CHECK-NEXT: [[LD_LANE_3:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_2]], 3, killed [[COPY3]] - ; CHECK-NEXT: $q0 = COPY [[LD_LANE_3]] - ; CHECK-NEXT: RET_ReallyLR implicit $q0 - %0:gpr64common = COPY $x0 - %1:gpr64common = COPY $x1 - %2:gpr64common = COPY $x2 - %3:gpr64common = COPY $x3 - %4:fpr128 = LDRQui %0, 0 - %5:fpr128 = LD1i32 %4, 1, killed %1 - %6:fpr128 = LD1i32 %5, 2, killed %2 - %7:fpr128 = LD1i32 %6, 3, killed %3 - $q0 = COPY %7 - RET_ReallyLR implicit $q0 - ---- -name: negative_pattern_multiple_users -body: | - bb.0.entry: - liveins: $x0, $x1, $x2, $x3, $x4 - - ; CHECK-LABEL: name: negative_pattern_multiple_users - ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0 - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2 - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3 - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4 - ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], killed [[COPY1]], 0, 1 - ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i32]], %subreg.ssub - ; CHECK-NEXT: [[LD_LANE_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, killed [[COPY2]] - ; CHECK-NEXT: [[LD_LANE_2:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_1]], 2, killed [[COPY3]] - ; CHECK-NEXT: [[LD_LANE_3:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_2]], 3, killed [[COPY4]] - ; CHECK-NEXT: $q0 = COPY [[LD_LANE_3]] - ; CHECK-NEXT: $q1 = COPY [[LD_LANE_2]] - ; CHECK-NEXT: RET_ReallyLR implicit $q0, implicit $q1 - %0:gpr64common = COPY $x0 - %1:gpr64common = COPY $x1 - %2:gpr64common = COPY $x2 - %3:gpr64common = COPY $x3 - %4:gpr64common = COPY $x4 - %5:fpr32 = LDRSroX %0, killed %1, 0, 1 - %6:fpr128 = SUBREG_TO_REG 0, killed %5, %subreg.ssub - %7:fpr128 = LD1i32 %6, 1, killed %2 - %8:fpr128 = LD1i32 %7, 2, killed %3 - %9:fpr128 = LD1i32 %8, 3, killed %4 - $q0 = COPY %9 - $q1 = COPY %8 - RET_ReallyLR implicit $q0, implicit $q1 diff --git a/llvm/test/CodeGen/AArch64/andcompare.ll b/llvm/test/CodeGen/AArch64/andcompare.ll index cbacd17..0e15b94 100644 --- a/llvm/test/CodeGen/AArch64/andcompare.ll +++ b/llvm/test/CodeGen/AArch64/andcompare.ll @@ -1,23 +1,23 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-none-elf -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SDISEL -; RUN: llc -mtriple=aarch64-none-elf -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,GISEL +; RUN: llc -mtriple=aarch64-none-elf -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64-none-elf -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI define i32 @and_eq_eq(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_eq_eq: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, eq -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_eq_eq: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_eq_eq: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, eq +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_eq_eq: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp eq i32 %s0, %s1 %c1 = icmp eq i32 %s2, %s3 @@ -27,21 +27,21 @@ entry: } define i32 @and_eq_ne(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_eq_ne: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, eq -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_eq_ne: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_eq_ne: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, eq +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_eq_ne: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp eq i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -51,21 +51,21 @@ entry: } define i32 @and_eq_ult(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_eq_ult: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, eq -; SDISEL-NEXT: cset w0, lo -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_eq_ult: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_eq_ult: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, eq +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_eq_ult: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp eq i32 %s0, %s1 %c1 = icmp ult i32 %s2, %s3 @@ -75,21 +75,21 @@ entry: } define i32 @and_eq_ule(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_eq_ule: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, eq -; SDISEL-NEXT: cset w0, ls -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_eq_ule: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ls -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_eq_ule: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, eq +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_eq_ule: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ls +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp eq i32 %s0, %s1 %c1 = icmp ule i32 %s2, %s3 @@ -99,21 +99,21 @@ entry: } define i32 @and_eq_ugt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_eq_ugt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, eq -; SDISEL-NEXT: cset w0, hi -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_eq_ugt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_eq_ugt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, eq +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_eq_ugt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp eq i32 %s0, %s1 %c1 = icmp ugt i32 %s2, %s3 @@ -123,21 +123,21 @@ entry: } define i32 @and_eq_uge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_eq_uge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, eq -; SDISEL-NEXT: cset w0, hs -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_eq_uge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hs -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_eq_uge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, eq +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_eq_uge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hs +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp eq i32 %s0, %s1 %c1 = icmp uge i32 %s2, %s3 @@ -147,21 +147,21 @@ entry: } define i32 @and_eq_slt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_eq_slt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, eq -; SDISEL-NEXT: cset w0, lt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_eq_slt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_eq_slt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, eq +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_eq_slt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp eq i32 %s0, %s1 %c1 = icmp slt i32 %s2, %s3 @@ -171,21 +171,21 @@ entry: } define i32 @and_eq_sle(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_eq_sle: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, eq -; SDISEL-NEXT: cset w0, le -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_eq_sle: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, le -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_eq_sle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, eq +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_eq_sle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, le +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp eq i32 %s0, %s1 %c1 = icmp sle i32 %s2, %s3 @@ -195,21 +195,21 @@ entry: } define i32 @and_eq_sgt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_eq_sgt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, eq -; SDISEL-NEXT: cset w0, gt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_eq_sgt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, gt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_eq_sgt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, eq +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_eq_sgt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, gt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp eq i32 %s0, %s1 %c1 = icmp sgt i32 %s2, %s3 @@ -219,21 +219,21 @@ entry: } define i32 @and_eq_sge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_eq_sge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #8, eq -; SDISEL-NEXT: cset w0, ge -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_eq_sge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ge -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_eq_sge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #8, eq +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_eq_sge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ge +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp eq i32 %s0, %s1 %c1 = icmp sge i32 %s2, %s3 @@ -243,21 +243,21 @@ entry: } define i32 @and_ne_eq(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ne_eq: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ne -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ne_eq: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ne_eq: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ne +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ne_eq: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ne i32 %s0, %s1 %c1 = icmp eq i32 %s2, %s3 @@ -267,21 +267,21 @@ entry: } define i32 @and_ne_ne(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ne_ne: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, ne -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ne_ne: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ne_ne: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, ne +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ne_ne: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ne i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -291,21 +291,21 @@ entry: } define i32 @and_ne_ult(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ne_ult: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, ne -; SDISEL-NEXT: cset w0, lo -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ne_ult: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ne_ult: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, ne +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ne_ult: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ne i32 %s0, %s1 %c1 = icmp ult i32 %s2, %s3 @@ -315,21 +315,21 @@ entry: } define i32 @and_ne_ule(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ne_ule: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, ne -; SDISEL-NEXT: cset w0, ls -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ne_ule: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ls -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ne_ule: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, ne +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ne_ule: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ls +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ne i32 %s0, %s1 %c1 = icmp ule i32 %s2, %s3 @@ -339,21 +339,21 @@ entry: } define i32 @and_ne_ugt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ne_ugt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ne -; SDISEL-NEXT: cset w0, hi -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ne_ugt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ne_ugt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ne +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ne_ugt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ne i32 %s0, %s1 %c1 = icmp ugt i32 %s2, %s3 @@ -363,21 +363,21 @@ entry: } define i32 @and_ne_uge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ne_uge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ne -; SDISEL-NEXT: cset w0, hs -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ne_uge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hs -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ne_uge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ne +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ne_uge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hs +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ne i32 %s0, %s1 %c1 = icmp uge i32 %s2, %s3 @@ -387,21 +387,21 @@ entry: } define i32 @and_ne_slt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ne_slt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ne -; SDISEL-NEXT: cset w0, lt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ne_slt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ne_slt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ne +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ne_slt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ne i32 %s0, %s1 %c1 = icmp slt i32 %s2, %s3 @@ -411,21 +411,21 @@ entry: } define i32 @and_ne_sle(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ne_sle: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ne -; SDISEL-NEXT: cset w0, le -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ne_sle: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, le -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ne_sle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ne +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ne_sle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, le +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ne i32 %s0, %s1 %c1 = icmp sle i32 %s2, %s3 @@ -435,21 +435,21 @@ entry: } define i32 @and_ne_sgt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ne_sgt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, ne -; SDISEL-NEXT: cset w0, gt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ne_sgt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, gt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ne_sgt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, ne +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ne_sgt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, gt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ne i32 %s0, %s1 %c1 = icmp sgt i32 %s2, %s3 @@ -459,21 +459,21 @@ entry: } define i32 @and_ne_sge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ne_sge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #8, ne -; SDISEL-NEXT: cset w0, ge -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ne_sge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ge -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ne_sge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #8, ne +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ne_sge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ge +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ne i32 %s0, %s1 %c1 = icmp sge i32 %s2, %s3 @@ -483,21 +483,21 @@ entry: } define i32 @and_ult_eq(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ult_eq: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lo -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ult_eq: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ult_eq: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lo +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ult_eq: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ult i32 %s0, %s1 %c1 = icmp eq i32 %s2, %s3 @@ -507,21 +507,21 @@ entry: } define i32 @and_ult_ne(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ult_ne: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, lo -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ult_ne: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ult_ne: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, lo +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ult_ne: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ult i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -531,21 +531,21 @@ entry: } define i32 @and_ult_ult(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ult_ult: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, lo -; SDISEL-NEXT: cset w0, lo -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ult_ult: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ult_ult: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, lo +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ult_ult: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ult i32 %s0, %s1 %c1 = icmp ult i32 %s2, %s3 @@ -555,21 +555,21 @@ entry: } define i32 @and_ult_ule(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ult_ule: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, lo -; SDISEL-NEXT: cset w0, ls -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ult_ule: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ls -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ult_ule: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, lo +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ult_ule: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ls +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ult i32 %s0, %s1 %c1 = icmp ule i32 %s2, %s3 @@ -579,21 +579,21 @@ entry: } define i32 @and_ult_ugt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ult_ugt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lo -; SDISEL-NEXT: cset w0, hi -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ult_ugt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ult_ugt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lo +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ult_ugt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ult i32 %s0, %s1 %c1 = icmp ugt i32 %s2, %s3 @@ -603,21 +603,21 @@ entry: } define i32 @and_ult_uge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ult_uge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lo -; SDISEL-NEXT: cset w0, hs -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ult_uge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hs -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ult_uge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lo +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ult_uge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hs +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ult i32 %s0, %s1 %c1 = icmp uge i32 %s2, %s3 @@ -627,21 +627,21 @@ entry: } define i32 @and_ult_slt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ult_slt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lo -; SDISEL-NEXT: cset w0, lt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ult_slt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ult_slt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lo +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ult_slt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ult i32 %s0, %s1 %c1 = icmp slt i32 %s2, %s3 @@ -651,21 +651,21 @@ entry: } define i32 @and_ult_sle(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ult_sle: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lo -; SDISEL-NEXT: cset w0, le -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ult_sle: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, le -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ult_sle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lo +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ult_sle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, le +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ult i32 %s0, %s1 %c1 = icmp sle i32 %s2, %s3 @@ -675,21 +675,21 @@ entry: } define i32 @and_ult_sgt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ult_sgt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, lo -; SDISEL-NEXT: cset w0, gt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ult_sgt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, gt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ult_sgt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, lo +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ult_sgt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, gt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ult i32 %s0, %s1 %c1 = icmp sgt i32 %s2, %s3 @@ -699,21 +699,21 @@ entry: } define i32 @and_ult_sge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ult_sge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #8, lo -; SDISEL-NEXT: cset w0, ge -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ult_sge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ge -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ult_sge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #8, lo +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ult_sge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ge +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ult i32 %s0, %s1 %c1 = icmp sge i32 %s2, %s3 @@ -723,21 +723,21 @@ entry: } define i32 @and_ule_eq(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ule_eq: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ls -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ule_eq: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ule_eq: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ls +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ule_eq: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ule i32 %s0, %s1 %c1 = icmp eq i32 %s2, %s3 @@ -747,21 +747,21 @@ entry: } define i32 @and_ule_ne(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ule_ne: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, ls -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ule_ne: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ule_ne: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, ls +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ule_ne: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ule i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -771,21 +771,21 @@ entry: } define i32 @and_ule_ult(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ule_ult: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, ls -; SDISEL-NEXT: cset w0, lo -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ule_ult: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ule_ult: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, ls +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ule_ult: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ule i32 %s0, %s1 %c1 = icmp ult i32 %s2, %s3 @@ -795,21 +795,21 @@ entry: } define i32 @and_ule_ule(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ule_ule: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, ls -; SDISEL-NEXT: cset w0, ls -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ule_ule: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ls -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ule_ule: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, ls +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ule_ule: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ls +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ule i32 %s0, %s1 %c1 = icmp ule i32 %s2, %s3 @@ -819,21 +819,21 @@ entry: } define i32 @and_ule_ugt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ule_ugt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ls -; SDISEL-NEXT: cset w0, hi -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ule_ugt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ule_ugt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ls +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ule_ugt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ule i32 %s0, %s1 %c1 = icmp ugt i32 %s2, %s3 @@ -843,21 +843,21 @@ entry: } define i32 @and_ule_uge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ule_uge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ls -; SDISEL-NEXT: cset w0, hs -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ule_uge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hs -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ule_uge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ls +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ule_uge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hs +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ule i32 %s0, %s1 %c1 = icmp uge i32 %s2, %s3 @@ -867,21 +867,21 @@ entry: } define i32 @and_ule_slt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ule_slt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ls -; SDISEL-NEXT: cset w0, lt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ule_slt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ule_slt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ls +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ule_slt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ule i32 %s0, %s1 %c1 = icmp slt i32 %s2, %s3 @@ -891,21 +891,21 @@ entry: } define i32 @and_ule_sle(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ule_sle: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ls -; SDISEL-NEXT: cset w0, le -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ule_sle: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, le -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ule_sle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ls +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ule_sle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, le +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ule i32 %s0, %s1 %c1 = icmp sle i32 %s2, %s3 @@ -915,21 +915,21 @@ entry: } define i32 @and_ule_sgt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ule_sgt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, ls -; SDISEL-NEXT: cset w0, gt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ule_sgt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, gt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ule_sgt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, ls +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ule_sgt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, gt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ule i32 %s0, %s1 %c1 = icmp sgt i32 %s2, %s3 @@ -939,21 +939,21 @@ entry: } define i32 @and_ule_sge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ule_sge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #8, ls -; SDISEL-NEXT: cset w0, ge -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ule_sge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ge -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ule_sge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #8, ls +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ule_sge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ge +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ule i32 %s0, %s1 %c1 = icmp sge i32 %s2, %s3 @@ -963,21 +963,21 @@ entry: } define i32 @and_ugt_eq(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ugt_eq: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, hi -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ugt_eq: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ugt_eq: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, hi +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ugt_eq: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ugt i32 %s0, %s1 %c1 = icmp eq i32 %s2, %s3 @@ -987,21 +987,21 @@ entry: } define i32 @and_ugt_ne(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ugt_ne: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, hi -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ugt_ne: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ugt_ne: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, hi +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ugt_ne: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ugt i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -1011,21 +1011,21 @@ entry: } define i32 @and_ugt_ult(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ugt_ult: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, hi -; SDISEL-NEXT: cset w0, lo -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ugt_ult: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ugt_ult: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, hi +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ugt_ult: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ugt i32 %s0, %s1 %c1 = icmp ult i32 %s2, %s3 @@ -1035,21 +1035,21 @@ entry: } define i32 @and_ugt_ule(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ugt_ule: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, hi -; SDISEL-NEXT: cset w0, ls -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ugt_ule: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ls -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ugt_ule: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, hi +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ugt_ule: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ls +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ugt i32 %s0, %s1 %c1 = icmp ule i32 %s2, %s3 @@ -1059,21 +1059,21 @@ entry: } define i32 @and_ugt_ugt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ugt_ugt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, hi -; SDISEL-NEXT: cset w0, hi -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ugt_ugt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ugt_ugt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, hi +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ugt_ugt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ugt i32 %s0, %s1 %c1 = icmp ugt i32 %s2, %s3 @@ -1083,21 +1083,21 @@ entry: } define i32 @and_ugt_uge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ugt_uge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, hi -; SDISEL-NEXT: cset w0, hs -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ugt_uge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hs -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ugt_uge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, hi +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ugt_uge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hs +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ugt i32 %s0, %s1 %c1 = icmp uge i32 %s2, %s3 @@ -1107,21 +1107,21 @@ entry: } define i32 @and_ugt_slt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ugt_slt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, hi -; SDISEL-NEXT: cset w0, lt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ugt_slt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ugt_slt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, hi +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ugt_slt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ugt i32 %s0, %s1 %c1 = icmp slt i32 %s2, %s3 @@ -1131,21 +1131,21 @@ entry: } define i32 @and_ugt_sle(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ugt_sle: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, hi -; SDISEL-NEXT: cset w0, le -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ugt_sle: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, le -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ugt_sle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, hi +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ugt_sle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, le +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ugt i32 %s0, %s1 %c1 = icmp sle i32 %s2, %s3 @@ -1155,21 +1155,21 @@ entry: } define i32 @and_ugt_sgt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ugt_sgt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, hi -; SDISEL-NEXT: cset w0, gt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ugt_sgt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, gt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ugt_sgt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, hi +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ugt_sgt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, gt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ugt i32 %s0, %s1 %c1 = icmp sgt i32 %s2, %s3 @@ -1179,21 +1179,21 @@ entry: } define i32 @and_ugt_sge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_ugt_sge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #8, hi -; SDISEL-NEXT: cset w0, ge -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_ugt_sge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ge -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_ugt_sge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #8, hi +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_ugt_sge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ge +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp ugt i32 %s0, %s1 %c1 = icmp sge i32 %s2, %s3 @@ -1203,21 +1203,21 @@ entry: } define i32 @and_uge_eq(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_uge_eq: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, hs -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_uge_eq: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_uge_eq: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, hs +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_uge_eq: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp uge i32 %s0, %s1 %c1 = icmp eq i32 %s2, %s3 @@ -1227,21 +1227,21 @@ entry: } define i32 @and_uge_ne(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_uge_ne: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, hs -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_uge_ne: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_uge_ne: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, hs +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_uge_ne: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp uge i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -1251,21 +1251,21 @@ entry: } define i32 @and_uge_ult(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_uge_ult: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, hs -; SDISEL-NEXT: cset w0, lo -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_uge_ult: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_uge_ult: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, hs +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_uge_ult: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp uge i32 %s0, %s1 %c1 = icmp ult i32 %s2, %s3 @@ -1275,21 +1275,21 @@ entry: } define i32 @and_uge_ule(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_uge_ule: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, hs -; SDISEL-NEXT: cset w0, ls -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_uge_ule: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ls -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_uge_ule: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, hs +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_uge_ule: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ls +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp uge i32 %s0, %s1 %c1 = icmp ule i32 %s2, %s3 @@ -1299,21 +1299,21 @@ entry: } define i32 @and_uge_ugt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_uge_ugt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, hs -; SDISEL-NEXT: cset w0, hi -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_uge_ugt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_uge_ugt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, hs +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_uge_ugt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp uge i32 %s0, %s1 %c1 = icmp ugt i32 %s2, %s3 @@ -1323,21 +1323,21 @@ entry: } define i32 @and_uge_uge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_uge_uge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, hs -; SDISEL-NEXT: cset w0, hs -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_uge_uge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hs -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_uge_uge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, hs +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_uge_uge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hs +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp uge i32 %s0, %s1 %c1 = icmp uge i32 %s2, %s3 @@ -1347,21 +1347,21 @@ entry: } define i32 @and_uge_slt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_uge_slt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, hs -; SDISEL-NEXT: cset w0, lt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_uge_slt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_uge_slt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, hs +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_uge_slt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp uge i32 %s0, %s1 %c1 = icmp slt i32 %s2, %s3 @@ -1371,21 +1371,21 @@ entry: } define i32 @and_uge_sle(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_uge_sle: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, hs -; SDISEL-NEXT: cset w0, le -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_uge_sle: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, le -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_uge_sle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, hs +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_uge_sle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, le +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp uge i32 %s0, %s1 %c1 = icmp sle i32 %s2, %s3 @@ -1395,21 +1395,21 @@ entry: } define i32 @and_uge_sgt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_uge_sgt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, hs -; SDISEL-NEXT: cset w0, gt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_uge_sgt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, gt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_uge_sgt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, hs +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_uge_sgt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, gt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp uge i32 %s0, %s1 %c1 = icmp sgt i32 %s2, %s3 @@ -1419,21 +1419,21 @@ entry: } define i32 @and_uge_sge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_uge_sge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #8, hs -; SDISEL-NEXT: cset w0, ge -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_uge_sge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ge -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_uge_sge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #8, hs +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_uge_sge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ge +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp uge i32 %s0, %s1 %c1 = icmp sge i32 %s2, %s3 @@ -1443,21 +1443,21 @@ entry: } define i32 @and_slt_eq(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_slt_eq: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lt -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_slt_eq: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_slt_eq: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lt +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_slt_eq: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp slt i32 %s0, %s1 %c1 = icmp eq i32 %s2, %s3 @@ -1467,21 +1467,21 @@ entry: } define i32 @and_slt_ne(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_slt_ne: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, lt -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_slt_ne: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_slt_ne: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, lt +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_slt_ne: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp slt i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -1491,21 +1491,21 @@ entry: } define i32 @and_slt_ult(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_slt_ult: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, lt -; SDISEL-NEXT: cset w0, lo -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_slt_ult: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_slt_ult: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, lt +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_slt_ult: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp slt i32 %s0, %s1 %c1 = icmp ult i32 %s2, %s3 @@ -1515,21 +1515,21 @@ entry: } define i32 @and_slt_ule(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_slt_ule: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, lt -; SDISEL-NEXT: cset w0, ls -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_slt_ule: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ls -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_slt_ule: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, lt +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_slt_ule: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ls +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp slt i32 %s0, %s1 %c1 = icmp ule i32 %s2, %s3 @@ -1539,21 +1539,21 @@ entry: } define i32 @and_slt_ugt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_slt_ugt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lt -; SDISEL-NEXT: cset w0, hi -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_slt_ugt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_slt_ugt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lt +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_slt_ugt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp slt i32 %s0, %s1 %c1 = icmp ugt i32 %s2, %s3 @@ -1563,21 +1563,21 @@ entry: } define i32 @and_slt_uge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_slt_uge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lt -; SDISEL-NEXT: cset w0, hs -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_slt_uge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hs -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_slt_uge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lt +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_slt_uge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hs +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp slt i32 %s0, %s1 %c1 = icmp uge i32 %s2, %s3 @@ -1587,21 +1587,21 @@ entry: } define i32 @and_slt_slt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_slt_slt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lt -; SDISEL-NEXT: cset w0, lt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_slt_slt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_slt_slt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lt +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_slt_slt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp slt i32 %s0, %s1 %c1 = icmp slt i32 %s2, %s3 @@ -1611,21 +1611,21 @@ entry: } define i32 @and_slt_sle(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_slt_sle: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lt -; SDISEL-NEXT: cset w0, le -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_slt_sle: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, le -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_slt_sle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lt +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_slt_sle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, le +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp slt i32 %s0, %s1 %c1 = icmp sle i32 %s2, %s3 @@ -1635,21 +1635,21 @@ entry: } define i32 @and_slt_sgt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_slt_sgt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, lt -; SDISEL-NEXT: cset w0, gt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_slt_sgt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, gt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_slt_sgt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, lt +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_slt_sgt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, gt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp slt i32 %s0, %s1 %c1 = icmp sgt i32 %s2, %s3 @@ -1659,21 +1659,21 @@ entry: } define i32 @and_slt_sge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_slt_sge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #8, lt -; SDISEL-NEXT: cset w0, ge -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_slt_sge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ge -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_slt_sge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #8, lt +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_slt_sge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ge +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp slt i32 %s0, %s1 %c1 = icmp sge i32 %s2, %s3 @@ -1683,21 +1683,21 @@ entry: } define i32 @and_sle_eq(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sle_eq: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, le -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sle_eq: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, le -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sle_eq: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, le +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sle_eq: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, le +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sle i32 %s0, %s1 %c1 = icmp eq i32 %s2, %s3 @@ -1707,21 +1707,21 @@ entry: } define i32 @and_sle_ne(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sle_ne: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, le -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sle_ne: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, le -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sle_ne: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, le +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sle_ne: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, le +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sle i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -1731,21 +1731,21 @@ entry: } define i32 @and_sle_ult(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sle_ult: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, le -; SDISEL-NEXT: cset w0, lo -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sle_ult: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, le -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sle_ult: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, le +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sle_ult: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, le +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sle i32 %s0, %s1 %c1 = icmp ult i32 %s2, %s3 @@ -1755,21 +1755,21 @@ entry: } define i32 @and_sle_ule(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sle_ule: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, le -; SDISEL-NEXT: cset w0, ls -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sle_ule: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, le -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ls -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sle_ule: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, le +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sle_ule: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, le +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ls +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sle i32 %s0, %s1 %c1 = icmp ule i32 %s2, %s3 @@ -1779,21 +1779,21 @@ entry: } define i32 @and_sle_ugt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sle_ugt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, le -; SDISEL-NEXT: cset w0, hi -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sle_ugt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, le -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sle_ugt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, le +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sle_ugt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, le +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sle i32 %s0, %s1 %c1 = icmp ugt i32 %s2, %s3 @@ -1803,21 +1803,21 @@ entry: } define i32 @and_sle_uge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sle_uge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, le -; SDISEL-NEXT: cset w0, hs -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sle_uge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, le -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hs -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sle_uge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, le +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sle_uge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, le +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hs +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sle i32 %s0, %s1 %c1 = icmp uge i32 %s2, %s3 @@ -1827,21 +1827,21 @@ entry: } define i32 @and_sle_slt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sle_slt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, le -; SDISEL-NEXT: cset w0, lt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sle_slt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, le -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sle_slt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, le +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sle_slt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, le +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sle i32 %s0, %s1 %c1 = icmp slt i32 %s2, %s3 @@ -1851,21 +1851,21 @@ entry: } define i32 @and_sle_sle(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sle_sle: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, le -; SDISEL-NEXT: cset w0, le -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sle_sle: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, le -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, le -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sle_sle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, le +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sle_sle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, le +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, le +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sle i32 %s0, %s1 %c1 = icmp sle i32 %s2, %s3 @@ -1875,21 +1875,21 @@ entry: } define i32 @and_sle_sgt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sle_sgt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, le -; SDISEL-NEXT: cset w0, gt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sle_sgt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, le -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, gt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sle_sgt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, le +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sle_sgt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, le +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, gt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sle i32 %s0, %s1 %c1 = icmp sgt i32 %s2, %s3 @@ -1899,21 +1899,21 @@ entry: } define i32 @and_sle_sge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sle_sge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #8, le -; SDISEL-NEXT: cset w0, ge -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sle_sge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, le -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ge -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sle_sge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #8, le +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sle_sge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, le +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ge +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sle i32 %s0, %s1 %c1 = icmp sge i32 %s2, %s3 @@ -1923,21 +1923,21 @@ entry: } define i32 @and_sgt_eq(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sgt_eq: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, gt -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sgt_eq: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sgt_eq: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, gt +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sgt_eq: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp eq i32 %s2, %s3 @@ -1947,21 +1947,21 @@ entry: } define i32 @and_sgt_ne(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sgt_ne: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, gt -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sgt_ne: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sgt_ne: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, gt +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sgt_ne: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -1971,21 +1971,21 @@ entry: } define i32 @and_sgt_ult(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sgt_ult: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, gt -; SDISEL-NEXT: cset w0, lo -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sgt_ult: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sgt_ult: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, gt +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sgt_ult: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp ult i32 %s2, %s3 @@ -1995,21 +1995,21 @@ entry: } define i32 @and_sgt_ule(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sgt_ule: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, gt -; SDISEL-NEXT: cset w0, ls -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sgt_ule: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ls -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sgt_ule: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, gt +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sgt_ule: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ls +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp ule i32 %s2, %s3 @@ -2019,21 +2019,21 @@ entry: } define i32 @and_sgt_ugt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sgt_ugt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, gt -; SDISEL-NEXT: cset w0, hi -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sgt_ugt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sgt_ugt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, gt +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sgt_ugt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp ugt i32 %s2, %s3 @@ -2043,21 +2043,21 @@ entry: } define i32 @and_sgt_uge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sgt_uge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, gt -; SDISEL-NEXT: cset w0, hs -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sgt_uge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hs -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sgt_uge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, gt +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sgt_uge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hs +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp uge i32 %s2, %s3 @@ -2067,21 +2067,21 @@ entry: } define i32 @and_sgt_slt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sgt_slt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, gt -; SDISEL-NEXT: cset w0, lt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sgt_slt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sgt_slt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, gt +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sgt_slt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp slt i32 %s2, %s3 @@ -2091,21 +2091,21 @@ entry: } define i32 @and_sgt_sle(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sgt_sle: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, gt -; SDISEL-NEXT: cset w0, le -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sgt_sle: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, le -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sgt_sle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, gt +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sgt_sle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, le +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp sle i32 %s2, %s3 @@ -2115,21 +2115,21 @@ entry: } define i32 @and_sgt_sgt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sgt_sgt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, gt -; SDISEL-NEXT: cset w0, gt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sgt_sgt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, gt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sgt_sgt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, gt +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sgt_sgt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, gt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp sgt i32 %s2, %s3 @@ -2139,21 +2139,21 @@ entry: } define i32 @and_sgt_sge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sgt_sge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #8, gt -; SDISEL-NEXT: cset w0, ge -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sgt_sge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ge -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sgt_sge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #8, gt +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sgt_sge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ge +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp sge i32 %s2, %s3 @@ -2163,21 +2163,21 @@ entry: } define i32 @and_sge_eq(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sge_eq: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ge -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sge_eq: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sge_eq: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ge +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sge_eq: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sge i32 %s0, %s1 %c1 = icmp eq i32 %s2, %s3 @@ -2187,21 +2187,21 @@ entry: } define i32 @and_sge_ne(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sge_ne: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, ge -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sge_ne: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sge_ne: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, ge +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sge_ne: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sge i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -2211,21 +2211,21 @@ entry: } define i32 @and_sge_ult(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sge_ult: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, ge -; SDISEL-NEXT: cset w0, lo -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sge_ult: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sge_ult: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, ge +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sge_ult: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sge i32 %s0, %s1 %c1 = icmp ult i32 %s2, %s3 @@ -2235,21 +2235,21 @@ entry: } define i32 @and_sge_ule(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sge_ule: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, ge -; SDISEL-NEXT: cset w0, ls -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sge_ule: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ls -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sge_ule: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, ge +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sge_ule: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ls +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sge i32 %s0, %s1 %c1 = icmp ule i32 %s2, %s3 @@ -2259,21 +2259,21 @@ entry: } define i32 @and_sge_ugt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sge_ugt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ge -; SDISEL-NEXT: cset w0, hi -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sge_ugt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sge_ugt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ge +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sge_ugt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sge i32 %s0, %s1 %c1 = icmp ugt i32 %s2, %s3 @@ -2283,21 +2283,21 @@ entry: } define i32 @and_sge_uge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sge_uge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ge -; SDISEL-NEXT: cset w0, hs -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sge_uge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hs -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sge_uge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ge +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sge_uge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hs +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sge i32 %s0, %s1 %c1 = icmp uge i32 %s2, %s3 @@ -2307,21 +2307,21 @@ entry: } define i32 @and_sge_slt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sge_slt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ge -; SDISEL-NEXT: cset w0, lt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sge_slt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sge_slt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ge +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sge_slt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sge i32 %s0, %s1 %c1 = icmp slt i32 %s2, %s3 @@ -2331,21 +2331,21 @@ entry: } define i32 @and_sge_sle(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sge_sle: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, ge -; SDISEL-NEXT: cset w0, le -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sge_sle: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, le -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sge_sle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, ge +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sge_sle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, le +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sge i32 %s0, %s1 %c1 = icmp sle i32 %s2, %s3 @@ -2355,21 +2355,21 @@ entry: } define i32 @and_sge_sgt(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sge_sgt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #4, ge -; SDISEL-NEXT: cset w0, gt -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sge_sgt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, gt -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sge_sgt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #4, ge +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sge_sgt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, gt +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sge i32 %s0, %s1 %c1 = icmp sgt i32 %s2, %s3 @@ -2379,21 +2379,21 @@ entry: } define i32 @and_sge_sge(i32 %s0, i32 %s1, i32 %s2, i32 %s3) { -; SDISEL-LABEL: and_sge_sge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #8, ge -; SDISEL-NEXT: cset w0, ge -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_sge_sge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ge -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_sge_sge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #8, ge +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_sge_sge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ge +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %c0 = icmp sge i32 %s0, %s1 %c1 = icmp sge i32 %s2, %s3 @@ -2403,19 +2403,19 @@ entry: } define i32 @cmp_to_ands1(i32 %num) { -; SDISEL-LABEL: cmp_to_ands1: -; SDISEL: // %bb.0: -; SDISEL-NEXT: and w8, w0, #0xff -; SDISEL-NEXT: tst w0, #0xfe -; SDISEL-NEXT: csel w0, w8, wzr, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: cmp_to_ands1: -; GISEL: // %bb.0: -; GISEL-NEXT: and w8, w0, #0xff -; GISEL-NEXT: cmp w8, #1 -; GISEL-NEXT: csel w0, w8, wzr, hi -; GISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_to_ands1: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: and w8, w0, #0xff +; CHECK-SD-NEXT: tst w0, #0xfe +; CHECK-SD-NEXT: csel w0, w8, wzr, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmp_to_ands1: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: and w8, w0, #0xff +; CHECK-GI-NEXT: cmp w8, #1 +; CHECK-GI-NEXT: csel w0, w8, wzr, hi +; CHECK-GI-NEXT: ret %and = and i32 %num, 255 %cmp = icmp ugt i32 %and, 1 %r = select i1 %cmp, i32 %and, i32 0 @@ -2423,19 +2423,19 @@ define i32 @cmp_to_ands1(i32 %num) { } define i32 @cmp_to_ands2(i32 %num) { -; SDISEL-LABEL: cmp_to_ands2: -; SDISEL: // %bb.0: -; SDISEL-NEXT: and w8, w0, #0xfe -; SDISEL-NEXT: tst w0, #0xc0 -; SDISEL-NEXT: csel w0, w8, wzr, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: cmp_to_ands2: -; GISEL: // %bb.0: -; GISEL-NEXT: and w8, w0, #0xfe -; GISEL-NEXT: cmp w8, #63 -; GISEL-NEXT: csel w0, w8, wzr, hi -; GISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_to_ands2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: and w8, w0, #0xfe +; CHECK-SD-NEXT: tst w0, #0xc0 +; CHECK-SD-NEXT: csel w0, w8, wzr, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmp_to_ands2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: and w8, w0, #0xfe +; CHECK-GI-NEXT: cmp w8, #63 +; CHECK-GI-NEXT: csel w0, w8, wzr, hi +; CHECK-GI-NEXT: ret %and = and i32 %num, 254 %cmp = icmp ugt i32 %and, 63 %r = select i1 %cmp, i32 %and, i32 0 @@ -2443,19 +2443,19 @@ define i32 @cmp_to_ands2(i32 %num) { } define i32 @cmp_to_ands3(i32 %num, i32 %a) { -; SDISEL-LABEL: cmp_to_ands3: -; SDISEL: // %bb.0: -; SDISEL-NEXT: tst w0, #0x10 -; SDISEL-NEXT: csel w0, w1, wzr, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: cmp_to_ands3: -; GISEL: // %bb.0: -; GISEL-NEXT: mov w8, #23 // =0x17 -; GISEL-NEXT: and w8, w0, w8 -; GISEL-NEXT: cmp w8, #7 -; GISEL-NEXT: csel w0, w1, wzr, hi -; GISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_to_ands3: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: tst w0, #0x10 +; CHECK-SD-NEXT: csel w0, w1, wzr, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmp_to_ands3: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #23 // =0x17 +; CHECK-GI-NEXT: and w8, w0, w8 +; CHECK-GI-NEXT: cmp w8, #7 +; CHECK-GI-NEXT: csel w0, w1, wzr, hi +; CHECK-GI-NEXT: ret %and = and i32 %num, 23 %cmp = icmp ugt i32 %and, 7 %r = select i1 %cmp, i32 %a, i32 0 @@ -2463,19 +2463,19 @@ define i32 @cmp_to_ands3(i32 %num, i32 %a) { } define i32 @cmp_to_ands4(i32 %num, i32 %a) { -; SDISEL-LABEL: cmp_to_ands4: -; SDISEL: // %bb.0: -; SDISEL-NEXT: and w8, w0, #0x30 -; SDISEL-NEXT: tst w0, #0x20 -; SDISEL-NEXT: csel w0, w8, w1, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: cmp_to_ands4: -; GISEL: // %bb.0: -; GISEL-NEXT: and w8, w0, #0x30 -; GISEL-NEXT: cmp w8, #31 -; GISEL-NEXT: csel w0, w8, w1, ls -; GISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_to_ands4: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: and w8, w0, #0x30 +; CHECK-SD-NEXT: tst w0, #0x20 +; CHECK-SD-NEXT: csel w0, w8, w1, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmp_to_ands4: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: and w8, w0, #0x30 +; CHECK-GI-NEXT: cmp w8, #31 +; CHECK-GI-NEXT: csel w0, w8, w1, ls +; CHECK-GI-NEXT: ret %and = and i32 %num, 48 %cmp = icmp ule i32 %and, 31 %r = select i1 %cmp, i32 %and, i32 %a @@ -2483,19 +2483,19 @@ define i32 @cmp_to_ands4(i32 %num, i32 %a) { } define i32 @cmp_to_ands5(i32 %num, i32 %a) { -; SDISEL-LABEL: cmp_to_ands5: -; SDISEL: // %bb.0: -; SDISEL-NEXT: and w8, w0, #0xf8 -; SDISEL-NEXT: tst w0, #0xc0 -; SDISEL-NEXT: csel w0, w8, w1, eq -; SDISEL-NEXT: ret -; -; GISEL-LABEL: cmp_to_ands5: -; GISEL: // %bb.0: -; GISEL-NEXT: and w8, w0, #0xf8 -; GISEL-NEXT: cmp w8, #64 -; GISEL-NEXT: csel w0, w8, w1, lo -; GISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_to_ands5: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: and w8, w0, #0xf8 +; CHECK-SD-NEXT: tst w0, #0xc0 +; CHECK-SD-NEXT: csel w0, w8, w1, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmp_to_ands5: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: and w8, w0, #0xf8 +; CHECK-GI-NEXT: cmp w8, #64 +; CHECK-GI-NEXT: csel w0, w8, w1, lo +; CHECK-GI-NEXT: ret %and = and i32 %num, 248 %cmp = icmp ult i32 %and, 64 %r = select i1 %cmp, i32 %and, i32 %a @@ -2503,19 +2503,19 @@ define i32 @cmp_to_ands5(i32 %num, i32 %a) { } define i32 @cmp_to_ands6(i32 %num) { -; SDISEL-LABEL: cmp_to_ands6: -; SDISEL: // %bb.0: -; SDISEL-NEXT: and w8, w0, #0xfe -; SDISEL-NEXT: tst w0, #0xf0 -; SDISEL-NEXT: csel w0, w8, wzr, ne -; SDISEL-NEXT: ret -; -; GISEL-LABEL: cmp_to_ands6: -; GISEL: // %bb.0: -; GISEL-NEXT: and w8, w0, #0xfe -; GISEL-NEXT: cmp w8, #16 -; GISEL-NEXT: csel w0, w8, wzr, hs -; GISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_to_ands6: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: and w8, w0, #0xfe +; CHECK-SD-NEXT: tst w0, #0xf0 +; CHECK-SD-NEXT: csel w0, w8, wzr, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmp_to_ands6: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: and w8, w0, #0xfe +; CHECK-GI-NEXT: cmp w8, #16 +; CHECK-GI-NEXT: csel w0, w8, wzr, hs +; CHECK-GI-NEXT: ret %and = and i32 %num, 254 %cmp = icmp uge i32 %and, 16 %r = select i1 %cmp, i32 %and, i32 0 @@ -2523,21 +2523,21 @@ define i32 @cmp_to_ands6(i32 %num) { } define i1 @and_fcmp(float %0, float %1) { -; SDISEL-LABEL: and_fcmp: -; SDISEL: // %bb.0: -; SDISEL-NEXT: fcmp s1, s1 -; SDISEL-NEXT: fccmp s0, s0, #0, vs -; SDISEL-NEXT: cset w0, vs -; SDISEL-NEXT: ret -; -; GISEL-LABEL: and_fcmp: -; GISEL: // %bb.0: -; GISEL-NEXT: fcmp s0, #0.0 -; GISEL-NEXT: cset w8, vs -; GISEL-NEXT: fcmp s1, #0.0 -; GISEL-NEXT: cset w9, vs -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-SD-LABEL: and_fcmp: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: fcmp s1, s1 +; CHECK-SD-NEXT: fccmp s0, s0, #0, vs +; CHECK-SD-NEXT: cset w0, vs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: and_fcmp: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: fcmp s0, #0.0 +; CHECK-GI-NEXT: cset w8, vs +; CHECK-GI-NEXT: fcmp s1, #0.0 +; CHECK-GI-NEXT: cset w9, vs +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret %3 = fcmp uno float %0, 0.000000e+00 %4 = fcmp uno float %1, 0.000000e+00 diff --git a/llvm/test/CodeGen/AArch64/andorbrcompare.ll b/llvm/test/CodeGen/AArch64/andorbrcompare.ll index 951a5cd..5bc06ec 100644 --- a/llvm/test/CodeGen/AArch64/andorbrcompare.ll +++ b/llvm/test/CodeGen/AArch64/andorbrcompare.ll @@ -1,44 +1,44 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-none-elf -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SDISEL -; RUN: llc -mtriple=aarch64-none-elf -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,GISEL +; RUN: llc -mtriple=aarch64-none-elf -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64-none-elf -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI declare void @dummy() define i32 @and_eq_ne_ult(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, ptr %p) { -; SDISEL-LABEL: and_eq_ne_ult: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w2, w3 -; SDISEL-NEXT: ccmp w0, w1, #0, ne -; SDISEL-NEXT: b.eq .LBB0_3 -; SDISEL-NEXT: // %bb.1: // %entry -; SDISEL-NEXT: cmp w4, w5 -; SDISEL-NEXT: b.lo .LBB0_3 -; SDISEL-NEXT: // %bb.2: -; SDISEL-NEXT: mov w0, wzr -; SDISEL-NEXT: ret -; SDISEL-NEXT: .LBB0_3: // %if -; SDISEL-NEXT: mov w0, #1 // =0x1 -; SDISEL-NEXT: str w0, [x6] -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: and_eq_ne_ult: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w2, w3 +; CHECK-SD-NEXT: ccmp w0, w1, #0, ne +; CHECK-SD-NEXT: b.eq .LBB0_3 +; CHECK-SD-NEXT: // %bb.1: // %entry +; CHECK-SD-NEXT: cmp w4, w5 +; CHECK-SD-NEXT: b.lo .LBB0_3 +; CHECK-SD-NEXT: // %bb.2: +; CHECK-SD-NEXT: mov w0, wzr +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: .LBB0_3: // %if +; CHECK-SD-NEXT: mov w0, #1 // =0x1 +; CHECK-SD-NEXT: str w0, [x6] +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: and_eq_ne_ult: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: tbnz w8, #0, .LBB0_3 -; GISEL-NEXT: // %bb.1: // %entry -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: mov w0, wzr -; GISEL-NEXT: b.lo .LBB0_3 -; GISEL-NEXT: // %bb.2: // %common.ret -; GISEL-NEXT: ret -; GISEL-NEXT: .LBB0_3: // %if -; GISEL-NEXT: mov w0, #1 // =0x1 -; GISEL-NEXT: str w0, [x6] -; GISEL-NEXT: ret +; CHECK-GI-LABEL: and_eq_ne_ult: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w8, w8, w9 +; CHECK-GI-NEXT: tbnz w8, #0, .LBB0_3 +; CHECK-GI-NEXT: // %bb.1: // %entry +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: mov w0, wzr +; CHECK-GI-NEXT: b.lo .LBB0_3 +; CHECK-GI-NEXT: // %bb.2: // %common.ret +; CHECK-GI-NEXT: ret +; CHECK-GI-NEXT: .LBB0_3: // %if +; CHECK-GI-NEXT: mov w0, #1 // =0x1 +; CHECK-GI-NEXT: str w0, [x6] +; CHECK-GI-NEXT: ret entry: %c0 = icmp eq i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -56,40 +56,40 @@ else: } define i32 @and_ne_ult_ule(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, ptr %p) { -; SDISEL-LABEL: and_ne_ult_ule: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w2, w3 -; SDISEL-NEXT: ccmp w0, w1, #4, lo -; SDISEL-NEXT: b.ne .LBB1_3 -; SDISEL-NEXT: // %bb.1: // %entry -; SDISEL-NEXT: cmp w4, w5 -; SDISEL-NEXT: b.ls .LBB1_3 -; SDISEL-NEXT: // %bb.2: -; SDISEL-NEXT: mov w0, wzr -; SDISEL-NEXT: ret -; SDISEL-NEXT: .LBB1_3: // %if -; SDISEL-NEXT: mov w0, #1 // =0x1 -; SDISEL-NEXT: str w0, [x6] -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: and_ne_ult_ule: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w2, w3 +; CHECK-SD-NEXT: ccmp w0, w1, #4, lo +; CHECK-SD-NEXT: b.ne .LBB1_3 +; CHECK-SD-NEXT: // %bb.1: // %entry +; CHECK-SD-NEXT: cmp w4, w5 +; CHECK-SD-NEXT: b.ls .LBB1_3 +; CHECK-SD-NEXT: // %bb.2: +; CHECK-SD-NEXT: mov w0, wzr +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: .LBB1_3: // %if +; CHECK-SD-NEXT: mov w0, #1 // =0x1 +; CHECK-SD-NEXT: str w0, [x6] +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: and_ne_ult_ule: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: tbnz w8, #0, .LBB1_3 -; GISEL-NEXT: // %bb.1: // %entry -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: mov w0, wzr -; GISEL-NEXT: b.ls .LBB1_3 -; GISEL-NEXT: // %bb.2: // %common.ret -; GISEL-NEXT: ret -; GISEL-NEXT: .LBB1_3: // %if -; GISEL-NEXT: mov w0, #1 // =0x1 -; GISEL-NEXT: str w0, [x6] -; GISEL-NEXT: ret +; CHECK-GI-LABEL: and_ne_ult_ule: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w8, w8, w9 +; CHECK-GI-NEXT: tbnz w8, #0, .LBB1_3 +; CHECK-GI-NEXT: // %bb.1: // %entry +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: mov w0, wzr +; CHECK-GI-NEXT: b.ls .LBB1_3 +; CHECK-GI-NEXT: // %bb.2: // %common.ret +; CHECK-GI-NEXT: ret +; CHECK-GI-NEXT: .LBB1_3: // %if +; CHECK-GI-NEXT: mov w0, #1 // =0x1 +; CHECK-GI-NEXT: str w0, [x6] +; CHECK-GI-NEXT: ret entry: %c0 = icmp ne i32 %s0, %s1 %c1 = icmp ult i32 %s2, %s3 @@ -107,40 +107,40 @@ else: } define i32 @and_ult_ule_ugt(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, ptr %p) { -; SDISEL-LABEL: and_ult_ule_ugt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w2, w3 -; SDISEL-NEXT: ccmp w0, w1, #2, ls -; SDISEL-NEXT: b.lo .LBB2_3 -; SDISEL-NEXT: // %bb.1: // %entry -; SDISEL-NEXT: cmp w4, w5 -; SDISEL-NEXT: b.hi .LBB2_3 -; SDISEL-NEXT: // %bb.2: -; SDISEL-NEXT: mov w0, wzr -; SDISEL-NEXT: ret -; SDISEL-NEXT: .LBB2_3: // %if -; SDISEL-NEXT: mov w0, #1 // =0x1 -; SDISEL-NEXT: str w0, [x6] -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: and_ult_ule_ugt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w2, w3 +; CHECK-SD-NEXT: ccmp w0, w1, #2, ls +; CHECK-SD-NEXT: b.lo .LBB2_3 +; CHECK-SD-NEXT: // %bb.1: // %entry +; CHECK-SD-NEXT: cmp w4, w5 +; CHECK-SD-NEXT: b.hi .LBB2_3 +; CHECK-SD-NEXT: // %bb.2: +; CHECK-SD-NEXT: mov w0, wzr +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: .LBB2_3: // %if +; CHECK-SD-NEXT: mov w0, #1 // =0x1 +; CHECK-SD-NEXT: str w0, [x6] +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: and_ult_ule_ugt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ls -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: tbnz w8, #0, .LBB2_3 -; GISEL-NEXT: // %bb.1: // %entry -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: mov w0, wzr -; GISEL-NEXT: b.hi .LBB2_3 -; GISEL-NEXT: // %bb.2: // %common.ret -; GISEL-NEXT: ret -; GISEL-NEXT: .LBB2_3: // %if -; GISEL-NEXT: mov w0, #1 // =0x1 -; GISEL-NEXT: str w0, [x6] -; GISEL-NEXT: ret +; CHECK-GI-LABEL: and_ult_ule_ugt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ls +; CHECK-GI-NEXT: and w8, w8, w9 +; CHECK-GI-NEXT: tbnz w8, #0, .LBB2_3 +; CHECK-GI-NEXT: // %bb.1: // %entry +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: mov w0, wzr +; CHECK-GI-NEXT: b.hi .LBB2_3 +; CHECK-GI-NEXT: // %bb.2: // %common.ret +; CHECK-GI-NEXT: ret +; CHECK-GI-NEXT: .LBB2_3: // %if +; CHECK-GI-NEXT: mov w0, #1 // =0x1 +; CHECK-GI-NEXT: str w0, [x6] +; CHECK-GI-NEXT: ret entry: %c0 = icmp ult i32 %s0, %s1 %c1 = icmp ule i32 %s2, %s3 @@ -158,40 +158,40 @@ else: } define i32 @and_ule_ugt_uge(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, ptr %p) { -; SDISEL-LABEL: and_ule_ugt_uge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w2, w3 -; SDISEL-NEXT: ccmp w0, w1, #2, hi -; SDISEL-NEXT: b.ls .LBB3_3 -; SDISEL-NEXT: // %bb.1: // %entry -; SDISEL-NEXT: cmp w4, w5 -; SDISEL-NEXT: b.hs .LBB3_3 -; SDISEL-NEXT: // %bb.2: -; SDISEL-NEXT: mov w0, wzr -; SDISEL-NEXT: ret -; SDISEL-NEXT: .LBB3_3: // %if -; SDISEL-NEXT: mov w0, #1 // =0x1 -; SDISEL-NEXT: str w0, [x6] -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: and_ule_ugt_uge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w2, w3 +; CHECK-SD-NEXT: ccmp w0, w1, #2, hi +; CHECK-SD-NEXT: b.ls .LBB3_3 +; CHECK-SD-NEXT: // %bb.1: // %entry +; CHECK-SD-NEXT: cmp w4, w5 +; CHECK-SD-NEXT: b.hs .LBB3_3 +; CHECK-SD-NEXT: // %bb.2: +; CHECK-SD-NEXT: mov w0, wzr +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: .LBB3_3: // %if +; CHECK-SD-NEXT: mov w0, #1 // =0x1 +; CHECK-SD-NEXT: str w0, [x6] +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: and_ule_ugt_uge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: tbnz w8, #0, .LBB3_3 -; GISEL-NEXT: // %bb.1: // %entry -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: mov w0, wzr -; GISEL-NEXT: b.hs .LBB3_3 -; GISEL-NEXT: // %bb.2: // %common.ret -; GISEL-NEXT: ret -; GISEL-NEXT: .LBB3_3: // %if -; GISEL-NEXT: mov w0, #1 // =0x1 -; GISEL-NEXT: str w0, [x6] -; GISEL-NEXT: ret +; CHECK-GI-LABEL: and_ule_ugt_uge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w8, w8, w9 +; CHECK-GI-NEXT: tbnz w8, #0, .LBB3_3 +; CHECK-GI-NEXT: // %bb.1: // %entry +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: mov w0, wzr +; CHECK-GI-NEXT: b.hs .LBB3_3 +; CHECK-GI-NEXT: // %bb.2: // %common.ret +; CHECK-GI-NEXT: ret +; CHECK-GI-NEXT: .LBB3_3: // %if +; CHECK-GI-NEXT: mov w0, #1 // =0x1 +; CHECK-GI-NEXT: str w0, [x6] +; CHECK-GI-NEXT: ret entry: %c0 = icmp ule i32 %s0, %s1 %c1 = icmp ugt i32 %s2, %s3 @@ -209,40 +209,40 @@ else: } define i32 @and_ugt_uge_slt(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, ptr %p) { -; SDISEL-LABEL: and_ugt_uge_slt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w2, w3 -; SDISEL-NEXT: ccmp w0, w1, #0, hs -; SDISEL-NEXT: b.hi .LBB4_3 -; SDISEL-NEXT: // %bb.1: // %entry -; SDISEL-NEXT: cmp w4, w5 -; SDISEL-NEXT: b.lt .LBB4_3 -; SDISEL-NEXT: // %bb.2: -; SDISEL-NEXT: mov w0, wzr -; SDISEL-NEXT: ret -; SDISEL-NEXT: .LBB4_3: // %if -; SDISEL-NEXT: mov w0, #1 // =0x1 -; SDISEL-NEXT: str w0, [x6] -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: and_ugt_uge_slt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w2, w3 +; CHECK-SD-NEXT: ccmp w0, w1, #0, hs +; CHECK-SD-NEXT: b.hi .LBB4_3 +; CHECK-SD-NEXT: // %bb.1: // %entry +; CHECK-SD-NEXT: cmp w4, w5 +; CHECK-SD-NEXT: b.lt .LBB4_3 +; CHECK-SD-NEXT: // %bb.2: +; CHECK-SD-NEXT: mov w0, wzr +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: .LBB4_3: // %if +; CHECK-SD-NEXT: mov w0, #1 // =0x1 +; CHECK-SD-NEXT: str w0, [x6] +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: and_ugt_uge_slt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hs -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: tbnz w8, #0, .LBB4_3 -; GISEL-NEXT: // %bb.1: // %entry -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: mov w0, wzr -; GISEL-NEXT: b.lt .LBB4_3 -; GISEL-NEXT: // %bb.2: // %common.ret -; GISEL-NEXT: ret -; GISEL-NEXT: .LBB4_3: // %if -; GISEL-NEXT: mov w0, #1 // =0x1 -; GISEL-NEXT: str w0, [x6] -; GISEL-NEXT: ret +; CHECK-GI-LABEL: and_ugt_uge_slt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hs +; CHECK-GI-NEXT: and w8, w8, w9 +; CHECK-GI-NEXT: tbnz w8, #0, .LBB4_3 +; CHECK-GI-NEXT: // %bb.1: // %entry +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: mov w0, wzr +; CHECK-GI-NEXT: b.lt .LBB4_3 +; CHECK-GI-NEXT: // %bb.2: // %common.ret +; CHECK-GI-NEXT: ret +; CHECK-GI-NEXT: .LBB4_3: // %if +; CHECK-GI-NEXT: mov w0, #1 // =0x1 +; CHECK-GI-NEXT: str w0, [x6] +; CHECK-GI-NEXT: ret entry: %c0 = icmp ugt i32 %s0, %s1 %c1 = icmp uge i32 %s2, %s3 @@ -260,40 +260,40 @@ else: } define i32 @and_uge_slt_sle(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, ptr %p) { -; SDISEL-LABEL: and_uge_slt_sle: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w2, w3 -; SDISEL-NEXT: ccmp w0, w1, #0, lt -; SDISEL-NEXT: b.hs .LBB5_3 -; SDISEL-NEXT: // %bb.1: // %entry -; SDISEL-NEXT: cmp w4, w5 -; SDISEL-NEXT: b.le .LBB5_3 -; SDISEL-NEXT: // %bb.2: -; SDISEL-NEXT: mov w0, wzr -; SDISEL-NEXT: ret -; SDISEL-NEXT: .LBB5_3: // %if -; SDISEL-NEXT: mov w0, #1 // =0x1 -; SDISEL-NEXT: str w0, [x6] -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: and_uge_slt_sle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w2, w3 +; CHECK-SD-NEXT: ccmp w0, w1, #0, lt +; CHECK-SD-NEXT: b.hs .LBB5_3 +; CHECK-SD-NEXT: // %bb.1: // %entry +; CHECK-SD-NEXT: cmp w4, w5 +; CHECK-SD-NEXT: b.le .LBB5_3 +; CHECK-SD-NEXT: // %bb.2: +; CHECK-SD-NEXT: mov w0, wzr +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: .LBB5_3: // %if +; CHECK-SD-NEXT: mov w0, #1 // =0x1 +; CHECK-SD-NEXT: str w0, [x6] +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: and_uge_slt_sle: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, lt -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: tbnz w8, #0, .LBB5_3 -; GISEL-NEXT: // %bb.1: // %entry -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: mov w0, wzr -; GISEL-NEXT: b.le .LBB5_3 -; GISEL-NEXT: // %bb.2: // %common.ret -; GISEL-NEXT: ret -; GISEL-NEXT: .LBB5_3: // %if -; GISEL-NEXT: mov w0, #1 // =0x1 -; GISEL-NEXT: str w0, [x6] -; GISEL-NEXT: ret +; CHECK-GI-LABEL: and_uge_slt_sle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, lt +; CHECK-GI-NEXT: and w8, w8, w9 +; CHECK-GI-NEXT: tbnz w8, #0, .LBB5_3 +; CHECK-GI-NEXT: // %bb.1: // %entry +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: mov w0, wzr +; CHECK-GI-NEXT: b.le .LBB5_3 +; CHECK-GI-NEXT: // %bb.2: // %common.ret +; CHECK-GI-NEXT: ret +; CHECK-GI-NEXT: .LBB5_3: // %if +; CHECK-GI-NEXT: mov w0, #1 // =0x1 +; CHECK-GI-NEXT: str w0, [x6] +; CHECK-GI-NEXT: ret entry: %c0 = icmp uge i32 %s0, %s1 %c1 = icmp slt i32 %s2, %s3 @@ -311,40 +311,40 @@ else: } define i32 @and_slt_sle_sgt(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, ptr %p) { -; SDISEL-LABEL: and_slt_sle_sgt: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w2, w3 -; SDISEL-NEXT: ccmp w0, w1, #0, le -; SDISEL-NEXT: b.lt .LBB6_3 -; SDISEL-NEXT: // %bb.1: // %entry -; SDISEL-NEXT: cmp w4, w5 -; SDISEL-NEXT: b.gt .LBB6_3 -; SDISEL-NEXT: // %bb.2: -; SDISEL-NEXT: mov w0, wzr -; SDISEL-NEXT: ret -; SDISEL-NEXT: .LBB6_3: // %if -; SDISEL-NEXT: mov w0, #1 // =0x1 -; SDISEL-NEXT: str w0, [x6] -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: and_slt_sle_sgt: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w2, w3 +; CHECK-SD-NEXT: ccmp w0, w1, #0, le +; CHECK-SD-NEXT: b.lt .LBB6_3 +; CHECK-SD-NEXT: // %bb.1: // %entry +; CHECK-SD-NEXT: cmp w4, w5 +; CHECK-SD-NEXT: b.gt .LBB6_3 +; CHECK-SD-NEXT: // %bb.2: +; CHECK-SD-NEXT: mov w0, wzr +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: .LBB6_3: // %if +; CHECK-SD-NEXT: mov w0, #1 // =0x1 +; CHECK-SD-NEXT: str w0, [x6] +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: and_slt_sle_sgt: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, le -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: tbnz w8, #0, .LBB6_3 -; GISEL-NEXT: // %bb.1: // %entry -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: mov w0, wzr -; GISEL-NEXT: b.gt .LBB6_3 -; GISEL-NEXT: // %bb.2: // %common.ret -; GISEL-NEXT: ret -; GISEL-NEXT: .LBB6_3: // %if -; GISEL-NEXT: mov w0, #1 // =0x1 -; GISEL-NEXT: str w0, [x6] -; GISEL-NEXT: ret +; CHECK-GI-LABEL: and_slt_sle_sgt: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, le +; CHECK-GI-NEXT: and w8, w8, w9 +; CHECK-GI-NEXT: tbnz w8, #0, .LBB6_3 +; CHECK-GI-NEXT: // %bb.1: // %entry +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: mov w0, wzr +; CHECK-GI-NEXT: b.gt .LBB6_3 +; CHECK-GI-NEXT: // %bb.2: // %common.ret +; CHECK-GI-NEXT: ret +; CHECK-GI-NEXT: .LBB6_3: // %if +; CHECK-GI-NEXT: mov w0, #1 // =0x1 +; CHECK-GI-NEXT: str w0, [x6] +; CHECK-GI-NEXT: ret entry: %c0 = icmp slt i32 %s0, %s1 %c1 = icmp sle i32 %s2, %s3 @@ -362,40 +362,40 @@ else: } define i32 @and_sle_sgt_sge(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, ptr %p) { -; SDISEL-LABEL: and_sle_sgt_sge: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: cmp w2, w3 -; SDISEL-NEXT: ccmp w0, w1, #0, gt -; SDISEL-NEXT: b.le .LBB7_3 -; SDISEL-NEXT: // %bb.1: // %entry -; SDISEL-NEXT: cmp w4, w5 -; SDISEL-NEXT: b.ge .LBB7_3 -; SDISEL-NEXT: // %bb.2: -; SDISEL-NEXT: mov w0, wzr -; SDISEL-NEXT: ret -; SDISEL-NEXT: .LBB7_3: // %if -; SDISEL-NEXT: mov w0, #1 // =0x1 -; SDISEL-NEXT: str w0, [x6] -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: and_sle_sgt_sge: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmp w2, w3 +; CHECK-SD-NEXT: ccmp w0, w1, #0, gt +; CHECK-SD-NEXT: b.le .LBB7_3 +; CHECK-SD-NEXT: // %bb.1: // %entry +; CHECK-SD-NEXT: cmp w4, w5 +; CHECK-SD-NEXT: b.ge .LBB7_3 +; CHECK-SD-NEXT: // %bb.2: +; CHECK-SD-NEXT: mov w0, wzr +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: .LBB7_3: // %if +; CHECK-SD-NEXT: mov w0, #1 // =0x1 +; CHECK-SD-NEXT: str w0, [x6] +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: and_sle_sgt_sge: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, le -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, gt -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: tbnz w8, #0, .LBB7_3 -; GISEL-NEXT: // %bb.1: // %entry -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: mov w0, wzr -; GISEL-NEXT: b.ge .LBB7_3 -; GISEL-NEXT: // %bb.2: // %common.ret -; GISEL-NEXT: ret -; GISEL-NEXT: .LBB7_3: // %if -; GISEL-NEXT: mov w0, #1 // =0x1 -; GISEL-NEXT: str w0, [x6] -; GISEL-NEXT: ret +; CHECK-GI-LABEL: and_sle_sgt_sge: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, le +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, gt +; CHECK-GI-NEXT: and w8, w8, w9 +; CHECK-GI-NEXT: tbnz w8, #0, .LBB7_3 +; CHECK-GI-NEXT: // %bb.1: // %entry +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: mov w0, wzr +; CHECK-GI-NEXT: b.ge .LBB7_3 +; CHECK-GI-NEXT: // %bb.2: // %common.ret +; CHECK-GI-NEXT: ret +; CHECK-GI-NEXT: .LBB7_3: // %if +; CHECK-GI-NEXT: mov w0, #1 // =0x1 +; CHECK-GI-NEXT: str w0, [x6] +; CHECK-GI-NEXT: ret entry: %c0 = icmp sle i32 %s0, %s1 %c1 = icmp sgt i32 %s2, %s3 diff --git a/llvm/test/CodeGen/AArch64/arm64-ccmp.ll b/llvm/test/CodeGen/AArch64/arm64-ccmp.ll index 06e957f..a546ffd 100644 --- a/llvm/test/CodeGen/AArch64/arm64-ccmp.ll +++ b/llvm/test/CodeGen/AArch64/arm64-ccmp.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -debugify-and-strip-all-safe -mcpu=cyclone -verify-machineinstrs -aarch64-enable-ccmp -aarch64-stress-ccmp | FileCheck %s --check-prefixes=CHECK,SDISEL -; RUN: llc < %s -debugify-and-strip-all-safe -mcpu=cyclone -verify-machineinstrs -aarch64-enable-ccmp -aarch64-stress-ccmp -global-isel | FileCheck %s --check-prefixes=CHECK,GISEL +; RUN: llc < %s -debugify-and-strip-all-safe -mcpu=cyclone -verify-machineinstrs -aarch64-enable-ccmp -aarch64-stress-ccmp | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc < %s -debugify-and-strip-all-safe -mcpu=cyclone -verify-machineinstrs -aarch64-enable-ccmp -aarch64-stress-ccmp -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI target triple = "arm64-apple-ios" define i32 @single_same(i32 %a, i32 %b) nounwind ssp { @@ -32,31 +32,31 @@ if.end: ; Different condition codes for the two compares. define i32 @single_different(i32 %a, i32 %b) nounwind ssp { -; SDISEL-LABEL: single_different: -; SDISEL: ; %bb.0: ; %entry -; SDISEL-NEXT: cmp w0, #6 -; SDISEL-NEXT: ccmp w1, #17, #0, ge -; SDISEL-NEXT: b.eq LBB1_2 -; SDISEL-NEXT: ; %bb.1: ; %if.then -; SDISEL-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill -; SDISEL-NEXT: bl _foo -; SDISEL-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload -; SDISEL-NEXT: LBB1_2: ; %if.end -; SDISEL-NEXT: mov w0, #7 ; =0x7 -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: single_different: +; CHECK-SD: ; %bb.0: ; %entry +; CHECK-SD-NEXT: cmp w0, #6 +; CHECK-SD-NEXT: ccmp w1, #17, #0, ge +; CHECK-SD-NEXT: b.eq LBB1_2 +; CHECK-SD-NEXT: ; %bb.1: ; %if.then +; CHECK-SD-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill +; CHECK-SD-NEXT: bl _foo +; CHECK-SD-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload +; CHECK-SD-NEXT: LBB1_2: ; %if.end +; CHECK-SD-NEXT: mov w0, #7 ; =0x7 +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: single_different: -; GISEL: ; %bb.0: ; %entry -; GISEL-NEXT: cmp w0, #5 -; GISEL-NEXT: ccmp w1, #17, #0, gt -; GISEL-NEXT: b.eq LBB1_2 -; GISEL-NEXT: ; %bb.1: ; %if.then -; GISEL-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill -; GISEL-NEXT: bl _foo -; GISEL-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload -; GISEL-NEXT: LBB1_2: ; %if.end -; GISEL-NEXT: mov w0, #7 ; =0x7 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: single_different: +; CHECK-GI: ; %bb.0: ; %entry +; CHECK-GI-NEXT: cmp w0, #5 +; CHECK-GI-NEXT: ccmp w1, #17, #0, gt +; CHECK-GI-NEXT: b.eq LBB1_2 +; CHECK-GI-NEXT: ; %bb.1: ; %if.then +; CHECK-GI-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill +; CHECK-GI-NEXT: bl _foo +; CHECK-GI-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload +; CHECK-GI-NEXT: LBB1_2: ; %if.end +; CHECK-GI-NEXT: mov w0, #7 ; =0x7 +; CHECK-GI-NEXT: ret entry: %cmp = icmp sle i32 %a, 5 %cmp1 = icmp ne i32 %b, 17 @@ -73,41 +73,41 @@ if.end: ; Second block clobbers the flags, can't convert (easily). define i32 @single_flagclobber(i32 %a, i32 %b) nounwind ssp { -; SDISEL-LABEL: single_flagclobber: -; SDISEL: ; %bb.0: ; %entry -; SDISEL-NEXT: cmp w0, #5 -; SDISEL-NEXT: b.eq LBB2_2 -; SDISEL-NEXT: ; %bb.1: ; %lor.lhs.false -; SDISEL-NEXT: lsl w8, w1, #1 -; SDISEL-NEXT: cmp w1, #7 -; SDISEL-NEXT: csinc w8, w8, w1, lt -; SDISEL-NEXT: cmp w8, #16 -; SDISEL-NEXT: b.gt LBB2_3 -; SDISEL-NEXT: LBB2_2: ; %if.then -; SDISEL-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill -; SDISEL-NEXT: bl _foo -; SDISEL-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload -; SDISEL-NEXT: LBB2_3: ; %if.end -; SDISEL-NEXT: mov w0, #7 ; =0x7 -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: single_flagclobber: +; CHECK-SD: ; %bb.0: ; %entry +; CHECK-SD-NEXT: cmp w0, #5 +; CHECK-SD-NEXT: b.eq LBB2_2 +; CHECK-SD-NEXT: ; %bb.1: ; %lor.lhs.false +; CHECK-SD-NEXT: lsl w8, w1, #1 +; CHECK-SD-NEXT: cmp w1, #7 +; CHECK-SD-NEXT: csinc w8, w8, w1, lt +; CHECK-SD-NEXT: cmp w8, #16 +; CHECK-SD-NEXT: b.gt LBB2_3 +; CHECK-SD-NEXT: LBB2_2: ; %if.then +; CHECK-SD-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill +; CHECK-SD-NEXT: bl _foo +; CHECK-SD-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload +; CHECK-SD-NEXT: LBB2_3: ; %if.end +; CHECK-SD-NEXT: mov w0, #7 ; =0x7 +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: single_flagclobber: -; GISEL: ; %bb.0: ; %entry -; GISEL-NEXT: cmp w0, #5 -; GISEL-NEXT: b.eq LBB2_2 -; GISEL-NEXT: ; %bb.1: ; %lor.lhs.false -; GISEL-NEXT: lsl w8, w1, #1 -; GISEL-NEXT: cmp w1, #7 -; GISEL-NEXT: csinc w8, w8, w1, lt -; GISEL-NEXT: cmp w8, #17 -; GISEL-NEXT: b.ge LBB2_3 -; GISEL-NEXT: LBB2_2: ; %if.then -; GISEL-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill -; GISEL-NEXT: bl _foo -; GISEL-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload -; GISEL-NEXT: LBB2_3: ; %if.end -; GISEL-NEXT: mov w0, #7 ; =0x7 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: single_flagclobber: +; CHECK-GI: ; %bb.0: ; %entry +; CHECK-GI-NEXT: cmp w0, #5 +; CHECK-GI-NEXT: b.eq LBB2_2 +; CHECK-GI-NEXT: ; %bb.1: ; %lor.lhs.false +; CHECK-GI-NEXT: lsl w8, w1, #1 +; CHECK-GI-NEXT: cmp w1, #7 +; CHECK-GI-NEXT: csinc w8, w8, w1, lt +; CHECK-GI-NEXT: cmp w8, #17 +; CHECK-GI-NEXT: b.ge LBB2_3 +; CHECK-GI-NEXT: LBB2_2: ; %if.then +; CHECK-GI-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill +; CHECK-GI-NEXT: bl _foo +; CHECK-GI-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload +; CHECK-GI-NEXT: LBB2_3: ; %if.end +; CHECK-GI-NEXT: mov w0, #7 ; =0x7 +; CHECK-GI-NEXT: ret entry: %cmp = icmp eq i32 %a, 5 br i1 %cmp, label %if.then, label %lor.lhs.false @@ -171,37 +171,37 @@ if.end: ; preds = %if.then, %lor.lhs.f ; The sdiv/udiv instructions do not trap when the divisor is zero, so they are ; safe to speculate. define i32 @speculate_division(i32 %a, i32 %b) nounwind ssp { -; SDISEL-LABEL: speculate_division: -; SDISEL: ; %bb.0: ; %entry -; SDISEL-NEXT: cmp w0, #1 -; SDISEL-NEXT: sdiv w8, w1, w0 -; SDISEL-NEXT: ccmp w8, #16, #0, ge -; SDISEL-NEXT: b.le LBB4_2 -; SDISEL-NEXT: ; %bb.1: ; %if.end -; SDISEL-NEXT: mov w0, #7 ; =0x7 -; SDISEL-NEXT: ret -; SDISEL-NEXT: LBB4_2: ; %if.then -; SDISEL-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill -; SDISEL-NEXT: bl _foo -; SDISEL-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload -; SDISEL-NEXT: mov w0, #7 ; =0x7 -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: speculate_division: +; CHECK-SD: ; %bb.0: ; %entry +; CHECK-SD-NEXT: cmp w0, #1 +; CHECK-SD-NEXT: sdiv w8, w1, w0 +; CHECK-SD-NEXT: ccmp w8, #16, #0, ge +; CHECK-SD-NEXT: b.le LBB4_2 +; CHECK-SD-NEXT: ; %bb.1: ; %if.end +; CHECK-SD-NEXT: mov w0, #7 ; =0x7 +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: LBB4_2: ; %if.then +; CHECK-SD-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill +; CHECK-SD-NEXT: bl _foo +; CHECK-SD-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload +; CHECK-SD-NEXT: mov w0, #7 ; =0x7 +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: speculate_division: -; GISEL: ; %bb.0: ; %entry -; GISEL-NEXT: cmp w0, #0 -; GISEL-NEXT: sdiv w8, w1, w0 -; GISEL-NEXT: ccmp w8, #17, #0, gt -; GISEL-NEXT: b.lt LBB4_2 -; GISEL-NEXT: ; %bb.1: ; %if.end -; GISEL-NEXT: mov w0, #7 ; =0x7 -; GISEL-NEXT: ret -; GISEL-NEXT: LBB4_2: ; %if.then -; GISEL-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill -; GISEL-NEXT: bl _foo -; GISEL-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload -; GISEL-NEXT: mov w0, #7 ; =0x7 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: speculate_division: +; CHECK-GI: ; %bb.0: ; %entry +; CHECK-GI-NEXT: cmp w0, #0 +; CHECK-GI-NEXT: sdiv w8, w1, w0 +; CHECK-GI-NEXT: ccmp w8, #17, #0, gt +; CHECK-GI-NEXT: b.lt LBB4_2 +; CHECK-GI-NEXT: ; %bb.1: ; %if.end +; CHECK-GI-NEXT: mov w0, #7 ; =0x7 +; CHECK-GI-NEXT: ret +; CHECK-GI-NEXT: LBB4_2: ; %if.then +; CHECK-GI-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill +; CHECK-GI-NEXT: bl _foo +; CHECK-GI-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload +; CHECK-GI-NEXT: mov w0, #7 ; =0x7 +; CHECK-GI-NEXT: ret entry: %cmp = icmp sgt i32 %a, 0 br i1 %cmp, label %land.lhs.true, label %if.end @@ -221,41 +221,41 @@ if.end: ; Floating point compare. define i32 @single_fcmp(i32 %a, float %b) nounwind ssp { -; SDISEL-LABEL: single_fcmp: -; SDISEL: ; %bb.0: ; %entry -; SDISEL-NEXT: cmp w0, #1 -; SDISEL-NEXT: scvtf s1, w0 -; SDISEL-NEXT: fdiv s0, s0, s1 -; SDISEL-NEXT: fmov s1, #17.00000000 -; SDISEL-NEXT: fccmp s0, s1, #8, ge -; SDISEL-NEXT: b.ge LBB5_2 -; SDISEL-NEXT: ; %bb.1: ; %if.end -; SDISEL-NEXT: mov w0, #7 ; =0x7 -; SDISEL-NEXT: ret -; SDISEL-NEXT: LBB5_2: ; %if.then -; SDISEL-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill -; SDISEL-NEXT: bl _foo -; SDISEL-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload -; SDISEL-NEXT: mov w0, #7 ; =0x7 -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: single_fcmp: +; CHECK-SD: ; %bb.0: ; %entry +; CHECK-SD-NEXT: cmp w0, #1 +; CHECK-SD-NEXT: scvtf s1, w0 +; CHECK-SD-NEXT: fdiv s0, s0, s1 +; CHECK-SD-NEXT: fmov s1, #17.00000000 +; CHECK-SD-NEXT: fccmp s0, s1, #8, ge +; CHECK-SD-NEXT: b.ge LBB5_2 +; CHECK-SD-NEXT: ; %bb.1: ; %if.end +; CHECK-SD-NEXT: mov w0, #7 ; =0x7 +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: LBB5_2: ; %if.then +; CHECK-SD-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill +; CHECK-SD-NEXT: bl _foo +; CHECK-SD-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload +; CHECK-SD-NEXT: mov w0, #7 ; =0x7 +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: single_fcmp: -; GISEL: ; %bb.0: ; %entry -; GISEL-NEXT: cmp w0, #0 -; GISEL-NEXT: scvtf s1, w0 -; GISEL-NEXT: fdiv s0, s0, s1 -; GISEL-NEXT: fmov s1, #17.00000000 -; GISEL-NEXT: fccmp s0, s1, #8, gt -; GISEL-NEXT: b.ge LBB5_2 -; GISEL-NEXT: ; %bb.1: ; %if.end -; GISEL-NEXT: mov w0, #7 ; =0x7 -; GISEL-NEXT: ret -; GISEL-NEXT: LBB5_2: ; %if.then -; GISEL-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill -; GISEL-NEXT: bl _foo -; GISEL-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload -; GISEL-NEXT: mov w0, #7 ; =0x7 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: single_fcmp: +; CHECK-GI: ; %bb.0: ; %entry +; CHECK-GI-NEXT: cmp w0, #0 +; CHECK-GI-NEXT: scvtf s1, w0 +; CHECK-GI-NEXT: fdiv s0, s0, s1 +; CHECK-GI-NEXT: fmov s1, #17.00000000 +; CHECK-GI-NEXT: fccmp s0, s1, #8, gt +; CHECK-GI-NEXT: b.ge LBB5_2 +; CHECK-GI-NEXT: ; %bb.1: ; %if.end +; CHECK-GI-NEXT: mov w0, #7 ; =0x7 +; CHECK-GI-NEXT: ret +; CHECK-GI-NEXT: LBB5_2: ; %if.then +; CHECK-GI-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill +; CHECK-GI-NEXT: bl _foo +; CHECK-GI-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload +; CHECK-GI-NEXT: mov w0, #7 ; =0x7 +; CHECK-GI-NEXT: ret entry: %cmp = icmp sgt i32 %a, 0 br i1 %cmp, label %land.lhs.true, label %if.end @@ -499,28 +499,28 @@ define float @select_or_float(i32 %w0, i32 %w1, float %x2, float %x3) { } define i64 @gccbug(i64 %x0, i64 %x1) { -; SDISEL-LABEL: gccbug: -; SDISEL: ; %bb.0: -; SDISEL-NEXT: cmp x0, #2 -; SDISEL-NEXT: ccmp x0, #4, #4, ne -; SDISEL-NEXT: ccmp x1, #0, #0, eq -; SDISEL-NEXT: mov w8, #1 ; =0x1 -; SDISEL-NEXT: cinc x0, x8, eq -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: gccbug: +; CHECK-SD: ; %bb.0: +; CHECK-SD-NEXT: cmp x0, #2 +; CHECK-SD-NEXT: ccmp x0, #4, #4, ne +; CHECK-SD-NEXT: ccmp x1, #0, #0, eq +; CHECK-SD-NEXT: mov w8, #1 ; =0x1 +; CHECK-SD-NEXT: cinc x0, x8, eq +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: gccbug: -; GISEL: ; %bb.0: -; GISEL-NEXT: cmp x1, #0 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmp x0, #2 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: cmp x0, #4 -; GISEL-NEXT: cset w10, eq -; GISEL-NEXT: orr w9, w10, w9 -; GISEL-NEXT: and w8, w9, w8 -; GISEL-NEXT: and x8, x8, #0x1 -; GISEL-NEXT: add x0, x8, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: gccbug: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: cmp x1, #0 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmp x0, #2 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: cmp x0, #4 +; CHECK-GI-NEXT: cset w10, eq +; CHECK-GI-NEXT: orr w9, w10, w9 +; CHECK-GI-NEXT: and w8, w9, w8 +; CHECK-GI-NEXT: and x8, x8, #0x1 +; CHECK-GI-NEXT: add x0, x8, #1 +; CHECK-GI-NEXT: ret %cmp0 = icmp eq i64 %x1, 0 %cmp1 = icmp eq i64 %x0, 2 %cmp2 = icmp eq i64 %x0, 4 @@ -570,23 +570,23 @@ define i32 @select_andor(i32 %v1, i32 %v2, i32 %v3) { } define i32 @select_andor32(i32 %v1, i32 %v2, i32 %v3) { -; SDISEL-LABEL: select_andor32: -; SDISEL: ; %bb.0: -; SDISEL-NEXT: cmp w1, w2 -; SDISEL-NEXT: mov w8, #32 ; =0x20 -; SDISEL-NEXT: ccmp w0, w8, #4, lt -; SDISEL-NEXT: ccmp w0, w1, #0, eq -; SDISEL-NEXT: csel w0, w0, w1, eq -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: select_andor32: +; CHECK-SD: ; %bb.0: +; CHECK-SD-NEXT: cmp w1, w2 +; CHECK-SD-NEXT: mov w8, #32 ; =0x20 +; CHECK-SD-NEXT: ccmp w0, w8, #4, lt +; CHECK-SD-NEXT: ccmp w0, w1, #0, eq +; CHECK-SD-NEXT: csel w0, w0, w1, eq +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: select_andor32: -; GISEL: ; %bb.0: -; GISEL-NEXT: mov w8, #32 ; =0x20 -; GISEL-NEXT: cmp w1, w2 -; GISEL-NEXT: ccmp w0, w8, #4, lt -; GISEL-NEXT: ccmp w0, w1, #0, eq -; GISEL-NEXT: csel w0, w0, w1, eq -; GISEL-NEXT: ret +; CHECK-GI-LABEL: select_andor32: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: mov w8, #32 ; =0x20 +; CHECK-GI-NEXT: cmp w1, w2 +; CHECK-GI-NEXT: ccmp w0, w8, #4, lt +; CHECK-GI-NEXT: ccmp w0, w1, #0, eq +; CHECK-GI-NEXT: csel w0, w0, w1, eq +; CHECK-GI-NEXT: ret %c0 = icmp eq i32 %v1, %v2 %c1 = icmp sge i32 %v2, %v3 %c2 = icmp eq i32 %v1, 32 @@ -597,22 +597,22 @@ define i32 @select_andor32(i32 %v1, i32 %v2, i32 %v3) { } define i64 @select_noccmp1(i64 %v1, i64 %v2, i64 %v3, i64 %r) { -; SDISEL-LABEL: select_noccmp1: -; SDISEL: ; %bb.0: -; SDISEL-NEXT: cmp x0, #0 -; SDISEL-NEXT: ccmp x0, #13, #4, lt -; SDISEL-NEXT: cset w8, gt -; SDISEL-NEXT: cmp x2, #2 -; SDISEL-NEXT: ccmp x2, #4, #4, lt -; SDISEL-NEXT: csinc w8, w8, wzr, le -; SDISEL-NEXT: cmp w8, #0 -; SDISEL-NEXT: csel x0, xzr, x3, ne -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: select_noccmp1: +; CHECK-SD: ; %bb.0: +; CHECK-SD-NEXT: cmp x0, #0 +; CHECK-SD-NEXT: ccmp x0, #13, #4, lt +; CHECK-SD-NEXT: cset w8, gt +; CHECK-SD-NEXT: cmp x2, #2 +; CHECK-SD-NEXT: ccmp x2, #4, #4, lt +; CHECK-SD-NEXT: csinc w8, w8, wzr, le +; CHECK-SD-NEXT: cmp w8, #0 +; CHECK-SD-NEXT: csel x0, xzr, x3, ne +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: select_noccmp1: -; GISEL: ; %bb.0: -; GISEL-NEXT: mov x0, x3 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: select_noccmp1: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: mov x0, x3 +; CHECK-GI-NEXT: ret %c0 = icmp slt i64 %v1, 0 %c1 = icmp sgt i64 %v1, 13 %c2 = icmp slt i64 %v3, 2 @@ -627,28 +627,28 @@ define i64 @select_noccmp1(i64 %v1, i64 %v2, i64 %v3, i64 %r) { @g = global i32 0 define i64 @select_noccmp2(i64 %v1, i64 %v2, i64 %v3, i64 %r) { -; SDISEL-LABEL: select_noccmp2: -; SDISEL: ; %bb.0: -; SDISEL-NEXT: cmp x0, #0 -; SDISEL-NEXT: ccmp x0, #13, #0, ge -; SDISEL-NEXT: cset w8, gt -; SDISEL-NEXT: cmp w8, #0 -; SDISEL-NEXT: csel x0, xzr, x3, ne -; SDISEL-NEXT: sbfx w8, w8, #0, #1 -; SDISEL-NEXT: adrp x9, _g@PAGE -; SDISEL-NEXT: str w8, [x9, _g@PAGEOFF] -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: select_noccmp2: +; CHECK-SD: ; %bb.0: +; CHECK-SD-NEXT: cmp x0, #0 +; CHECK-SD-NEXT: ccmp x0, #13, #0, ge +; CHECK-SD-NEXT: cset w8, gt +; CHECK-SD-NEXT: cmp w8, #0 +; CHECK-SD-NEXT: csel x0, xzr, x3, ne +; CHECK-SD-NEXT: sbfx w8, w8, #0, #1 +; CHECK-SD-NEXT: adrp x9, _g@PAGE +; CHECK-SD-NEXT: str w8, [x9, _g@PAGEOFF] +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: select_noccmp2: -; GISEL: ; %bb.0: -; GISEL-NEXT: cmp x0, #14 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: tst w8, #0x1 -; GISEL-NEXT: csel x0, xzr, x3, ne -; GISEL-NEXT: sbfx w8, w8, #0, #1 -; GISEL-NEXT: adrp x9, _g@PAGE -; GISEL-NEXT: str w8, [x9, _g@PAGEOFF] -; GISEL-NEXT: ret +; CHECK-GI-LABEL: select_noccmp2: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: cmp x0, #14 +; CHECK-GI-NEXT: cset w8, hs +; CHECK-GI-NEXT: tst w8, #0x1 +; CHECK-GI-NEXT: csel x0, xzr, x3, ne +; CHECK-GI-NEXT: sbfx w8, w8, #0, #1 +; CHECK-GI-NEXT: adrp x9, _g@PAGE +; CHECK-GI-NEXT: str w8, [x9, _g@PAGEOFF] +; CHECK-GI-NEXT: ret %c0 = icmp slt i64 %v1, 0 %c1 = icmp sgt i64 %v1, 13 %or = or i1 %c0, %c1 @@ -661,33 +661,33 @@ define i64 @select_noccmp2(i64 %v1, i64 %v2, i64 %v3, i64 %r) { ; The following is not possible to implement with a single cmp;ccmp;csel ; sequence. define i32 @select_noccmp3(i32 %v0, i32 %v1, i32 %v2) { -; SDISEL-LABEL: select_noccmp3: -; SDISEL: ; %bb.0: -; SDISEL-NEXT: cmp w0, #0 -; SDISEL-NEXT: ccmp w0, #13, #0, ge -; SDISEL-NEXT: cset w8, gt -; SDISEL-NEXT: cmp w0, #22 -; SDISEL-NEXT: mov w9, #44 ; =0x2c -; SDISEL-NEXT: ccmp w0, w9, #0, ge -; SDISEL-NEXT: csel w8, wzr, w8, le -; SDISEL-NEXT: cmp w0, #99 -; SDISEL-NEXT: mov w9, #77 ; =0x4d -; SDISEL-NEXT: ccmp w0, w9, #4, ne -; SDISEL-NEXT: cset w9, eq -; SDISEL-NEXT: tst w8, w9 -; SDISEL-NEXT: csel w0, w1, w2, ne -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: select_noccmp3: +; CHECK-SD: ; %bb.0: +; CHECK-SD-NEXT: cmp w0, #0 +; CHECK-SD-NEXT: ccmp w0, #13, #0, ge +; CHECK-SD-NEXT: cset w8, gt +; CHECK-SD-NEXT: cmp w0, #22 +; CHECK-SD-NEXT: mov w9, #44 ; =0x2c +; CHECK-SD-NEXT: ccmp w0, w9, #0, ge +; CHECK-SD-NEXT: csel w8, wzr, w8, le +; CHECK-SD-NEXT: cmp w0, #99 +; CHECK-SD-NEXT: mov w9, #77 ; =0x4d +; CHECK-SD-NEXT: ccmp w0, w9, #4, ne +; CHECK-SD-NEXT: cset w9, eq +; CHECK-SD-NEXT: tst w8, w9 +; CHECK-SD-NEXT: csel w0, w1, w2, ne +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: select_noccmp3: -; GISEL: ; %bb.0: -; GISEL-NEXT: mov w8, #99 ; =0x63 -; GISEL-NEXT: sub w9, w0, #45 -; GISEL-NEXT: cmp w0, #77 -; GISEL-NEXT: ccmp w0, w8, #4, ne -; GISEL-NEXT: ccmn w9, #23, #2, eq -; GISEL-NEXT: ccmp w0, #14, #0, lo -; GISEL-NEXT: csel w0, w1, w2, hs -; GISEL-NEXT: ret +; CHECK-GI-LABEL: select_noccmp3: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: mov w8, #99 ; =0x63 +; CHECK-GI-NEXT: sub w9, w0, #45 +; CHECK-GI-NEXT: cmp w0, #77 +; CHECK-GI-NEXT: ccmp w0, w8, #4, ne +; CHECK-GI-NEXT: ccmn w9, #23, #2, eq +; CHECK-GI-NEXT: ccmp w0, #14, #0, lo +; CHECK-GI-NEXT: csel w0, w1, w2, hs +; CHECK-GI-NEXT: ret %c0 = icmp slt i32 %v0, 0 %c1 = icmp sgt i32 %v0, 13 %c2 = icmp slt i32 %v0, 22 @@ -864,27 +864,27 @@ define i32 @select_or_olt_ueq_ogt(double %v0, double %v1, double %v2, double %v3 ; Verify that we correctly promote f16. define i32 @half_select_and_olt_oge(half %v0, half %v1, half %v2, half %v3, i32 %a, i32 %b) #0 { -; SDISEL-LABEL: half_select_and_olt_oge: -; SDISEL: ; %bb.0: -; SDISEL-NEXT: fcvt s1, h1 -; SDISEL-NEXT: fcvt s0, h0 -; SDISEL-NEXT: fcmp s0, s1 -; SDISEL-NEXT: fcvt s0, h3 -; SDISEL-NEXT: fcvt s1, h2 -; SDISEL-NEXT: fccmp s1, s0, #8, mi -; SDISEL-NEXT: csel w0, w0, w1, ge -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: half_select_and_olt_oge: +; CHECK-SD: ; %bb.0: +; CHECK-SD-NEXT: fcvt s1, h1 +; CHECK-SD-NEXT: fcvt s0, h0 +; CHECK-SD-NEXT: fcmp s0, s1 +; CHECK-SD-NEXT: fcvt s0, h3 +; CHECK-SD-NEXT: fcvt s1, h2 +; CHECK-SD-NEXT: fccmp s1, s0, #8, mi +; CHECK-SD-NEXT: csel w0, w0, w1, ge +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: half_select_and_olt_oge: -; GISEL: ; %bb.0: -; GISEL-NEXT: fcvt s0, h0 -; GISEL-NEXT: fcvt s1, h1 -; GISEL-NEXT: fcvt s2, h2 -; GISEL-NEXT: fcvt s3, h3 -; GISEL-NEXT: fcmp s0, s1 -; GISEL-NEXT: fccmp s2, s3, #8, mi -; GISEL-NEXT: csel w0, w0, w1, ge -; GISEL-NEXT: ret +; CHECK-GI-LABEL: half_select_and_olt_oge: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: fcvt s0, h0 +; CHECK-GI-NEXT: fcvt s1, h1 +; CHECK-GI-NEXT: fcvt s2, h2 +; CHECK-GI-NEXT: fcvt s3, h3 +; CHECK-GI-NEXT: fcmp s0, s1 +; CHECK-GI-NEXT: fccmp s2, s3, #8, mi +; CHECK-GI-NEXT: csel w0, w0, w1, ge +; CHECK-GI-NEXT: ret %c0 = fcmp olt half %v0, %v1 %c1 = fcmp oge half %v2, %v3 %cr = and i1 %c1, %c0 @@ -893,29 +893,29 @@ define i32 @half_select_and_olt_oge(half %v0, half %v1, half %v2, half %v3, i32 } define i32 @half_select_and_olt_one(half %v0, half %v1, half %v2, half %v3, i32 %a, i32 %b) #0 { -; SDISEL-LABEL: half_select_and_olt_one: -; SDISEL: ; %bb.0: -; SDISEL-NEXT: fcvt s1, h1 -; SDISEL-NEXT: fcvt s0, h0 -; SDISEL-NEXT: fcmp s0, s1 -; SDISEL-NEXT: fcvt s0, h3 -; SDISEL-NEXT: fcvt s1, h2 -; SDISEL-NEXT: fccmp s1, s0, #4, mi -; SDISEL-NEXT: fccmp s1, s0, #1, ne -; SDISEL-NEXT: csel w0, w0, w1, vc -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: half_select_and_olt_one: +; CHECK-SD: ; %bb.0: +; CHECK-SD-NEXT: fcvt s1, h1 +; CHECK-SD-NEXT: fcvt s0, h0 +; CHECK-SD-NEXT: fcmp s0, s1 +; CHECK-SD-NEXT: fcvt s0, h3 +; CHECK-SD-NEXT: fcvt s1, h2 +; CHECK-SD-NEXT: fccmp s1, s0, #4, mi +; CHECK-SD-NEXT: fccmp s1, s0, #1, ne +; CHECK-SD-NEXT: csel w0, w0, w1, vc +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: half_select_and_olt_one: -; GISEL: ; %bb.0: -; GISEL-NEXT: fcvt s0, h0 -; GISEL-NEXT: fcvt s1, h1 -; GISEL-NEXT: fcvt s2, h2 -; GISEL-NEXT: fcvt s3, h3 -; GISEL-NEXT: fcmp s0, s1 -; GISEL-NEXT: fccmp s2, s3, #4, mi -; GISEL-NEXT: fccmp s2, s3, #1, ne -; GISEL-NEXT: csel w0, w0, w1, vc -; GISEL-NEXT: ret +; CHECK-GI-LABEL: half_select_and_olt_one: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: fcvt s0, h0 +; CHECK-GI-NEXT: fcvt s1, h1 +; CHECK-GI-NEXT: fcvt s2, h2 +; CHECK-GI-NEXT: fcvt s3, h3 +; CHECK-GI-NEXT: fcmp s0, s1 +; CHECK-GI-NEXT: fccmp s2, s3, #4, mi +; CHECK-GI-NEXT: fccmp s2, s3, #1, ne +; CHECK-GI-NEXT: csel w0, w0, w1, vc +; CHECK-GI-NEXT: ret %c0 = fcmp olt half %v0, %v1 %c1 = fcmp one half %v2, %v3 %cr = and i1 %c1, %c0 @@ -926,51 +926,51 @@ define i32 @half_select_and_olt_one(half %v0, half %v1, half %v2, half %v3, i32 ; Also verify that we don't try to generate f128 FCCMPs, using RT calls instead. define i32 @f128_select_and_olt_oge(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32 %b) #0 { -; SDISEL-LABEL: f128_select_and_olt_oge: -; SDISEL: ; %bb.0: -; SDISEL-NEXT: sub sp, sp, #80 -; SDISEL-NEXT: stp x22, x21, [sp, #32] ; 16-byte Folded Spill -; SDISEL-NEXT: stp x20, x19, [sp, #48] ; 16-byte Folded Spill -; SDISEL-NEXT: stp x29, x30, [sp, #64] ; 16-byte Folded Spill -; SDISEL-NEXT: mov x19, x1 -; SDISEL-NEXT: mov x20, x0 -; SDISEL-NEXT: stp q2, q3, [sp] ; 32-byte Folded Spill -; SDISEL-NEXT: bl ___lttf2 -; SDISEL-NEXT: cmp w0, #0 -; SDISEL-NEXT: cset w21, lt -; SDISEL-NEXT: ldp q0, q1, [sp] ; 32-byte Folded Reload -; SDISEL-NEXT: bl ___getf2 -; SDISEL-NEXT: cmp w0, #0 -; SDISEL-NEXT: cset w8, ge -; SDISEL-NEXT: tst w8, w21 -; SDISEL-NEXT: csel w0, w20, w19, ne -; SDISEL-NEXT: ldp x29, x30, [sp, #64] ; 16-byte Folded Reload -; SDISEL-NEXT: ldp x20, x19, [sp, #48] ; 16-byte Folded Reload -; SDISEL-NEXT: ldp x22, x21, [sp, #32] ; 16-byte Folded Reload -; SDISEL-NEXT: add sp, sp, #80 -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: f128_select_and_olt_oge: +; CHECK-SD: ; %bb.0: +; CHECK-SD-NEXT: sub sp, sp, #80 +; CHECK-SD-NEXT: stp x22, x21, [sp, #32] ; 16-byte Folded Spill +; CHECK-SD-NEXT: stp x20, x19, [sp, #48] ; 16-byte Folded Spill +; CHECK-SD-NEXT: stp x29, x30, [sp, #64] ; 16-byte Folded Spill +; CHECK-SD-NEXT: mov x19, x1 +; CHECK-SD-NEXT: mov x20, x0 +; CHECK-SD-NEXT: stp q2, q3, [sp] ; 32-byte Folded Spill +; CHECK-SD-NEXT: bl ___lttf2 +; CHECK-SD-NEXT: cmp w0, #0 +; CHECK-SD-NEXT: cset w21, lt +; CHECK-SD-NEXT: ldp q0, q1, [sp] ; 32-byte Folded Reload +; CHECK-SD-NEXT: bl ___getf2 +; CHECK-SD-NEXT: cmp w0, #0 +; CHECK-SD-NEXT: cset w8, ge +; CHECK-SD-NEXT: tst w8, w21 +; CHECK-SD-NEXT: csel w0, w20, w19, ne +; CHECK-SD-NEXT: ldp x29, x30, [sp, #64] ; 16-byte Folded Reload +; CHECK-SD-NEXT: ldp x20, x19, [sp, #48] ; 16-byte Folded Reload +; CHECK-SD-NEXT: ldp x22, x21, [sp, #32] ; 16-byte Folded Reload +; CHECK-SD-NEXT: add sp, sp, #80 +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: f128_select_and_olt_oge: -; GISEL: ; %bb.0: -; GISEL-NEXT: sub sp, sp, #80 -; GISEL-NEXT: stp x22, x21, [sp, #32] ; 16-byte Folded Spill -; GISEL-NEXT: stp x20, x19, [sp, #48] ; 16-byte Folded Spill -; GISEL-NEXT: stp x29, x30, [sp, #64] ; 16-byte Folded Spill -; GISEL-NEXT: stp q3, q2, [sp] ; 32-byte Folded Spill -; GISEL-NEXT: mov x19, x0 -; GISEL-NEXT: mov x20, x1 -; GISEL-NEXT: bl ___lttf2 -; GISEL-NEXT: mov x21, x0 -; GISEL-NEXT: ldp q1, q0, [sp] ; 32-byte Folded Reload -; GISEL-NEXT: bl ___getf2 -; GISEL-NEXT: cmp w21, #0 -; GISEL-NEXT: ccmp w0, #0, #8, lt -; GISEL-NEXT: csel w0, w19, w20, ge -; GISEL-NEXT: ldp x29, x30, [sp, #64] ; 16-byte Folded Reload -; GISEL-NEXT: ldp x20, x19, [sp, #48] ; 16-byte Folded Reload -; GISEL-NEXT: ldp x22, x21, [sp, #32] ; 16-byte Folded Reload -; GISEL-NEXT: add sp, sp, #80 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: f128_select_and_olt_oge: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: sub sp, sp, #80 +; CHECK-GI-NEXT: stp x22, x21, [sp, #32] ; 16-byte Folded Spill +; CHECK-GI-NEXT: stp x20, x19, [sp, #48] ; 16-byte Folded Spill +; CHECK-GI-NEXT: stp x29, x30, [sp, #64] ; 16-byte Folded Spill +; CHECK-GI-NEXT: stp q3, q2, [sp] ; 32-byte Folded Spill +; CHECK-GI-NEXT: mov x19, x0 +; CHECK-GI-NEXT: mov x20, x1 +; CHECK-GI-NEXT: bl ___lttf2 +; CHECK-GI-NEXT: mov x21, x0 +; CHECK-GI-NEXT: ldp q1, q0, [sp] ; 32-byte Folded Reload +; CHECK-GI-NEXT: bl ___getf2 +; CHECK-GI-NEXT: cmp w21, #0 +; CHECK-GI-NEXT: ccmp w0, #0, #8, lt +; CHECK-GI-NEXT: csel w0, w19, w20, ge +; CHECK-GI-NEXT: ldp x29, x30, [sp, #64] ; 16-byte Folded Reload +; CHECK-GI-NEXT: ldp x20, x19, [sp, #48] ; 16-byte Folded Reload +; CHECK-GI-NEXT: ldp x22, x21, [sp, #32] ; 16-byte Folded Reload +; CHECK-GI-NEXT: add sp, sp, #80 +; CHECK-GI-NEXT: ret %c0 = fcmp olt fp128 %v0, %v1 %c1 = fcmp oge fp128 %v2, %v3 %cr = and i1 %c1, %c0 @@ -1048,46 +1048,46 @@ define i32 @deep_or2(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i32 %x, i32 %y) { ; This test is trying to test that multiple ccmp's don't get created in a way ; that they would have multiple uses. It doesn't seem to. define i32 @multiccmp(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %x, i32 %y) #0 { -; SDISEL-LABEL: multiccmp: -; SDISEL: ; %bb.0: ; %entry -; SDISEL-NEXT: stp x22, x21, [sp, #-48]! ; 16-byte Folded Spill -; SDISEL-NEXT: stp x20, x19, [sp, #16] ; 16-byte Folded Spill -; SDISEL-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill -; SDISEL-NEXT: mov x19, x5 -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: cset w20, gt -; SDISEL-NEXT: cmp w2, w3 -; SDISEL-NEXT: cset w21, ne -; SDISEL-NEXT: tst w20, w21 -; SDISEL-NEXT: csel w0, w5, w4, ne -; SDISEL-NEXT: bl _callee -; SDISEL-NEXT: tst w20, w21 -; SDISEL-NEXT: csel w0, w0, w19, ne -; SDISEL-NEXT: bl _callee -; SDISEL-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload -; SDISEL-NEXT: ldp x20, x19, [sp, #16] ; 16-byte Folded Reload -; SDISEL-NEXT: ldp x22, x21, [sp], #48 ; 16-byte Folded Reload -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: multiccmp: +; CHECK-SD: ; %bb.0: ; %entry +; CHECK-SD-NEXT: stp x22, x21, [sp, #-48]! ; 16-byte Folded Spill +; CHECK-SD-NEXT: stp x20, x19, [sp, #16] ; 16-byte Folded Spill +; CHECK-SD-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill +; CHECK-SD-NEXT: mov x19, x5 +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: cset w20, gt +; CHECK-SD-NEXT: cmp w2, w3 +; CHECK-SD-NEXT: cset w21, ne +; CHECK-SD-NEXT: tst w20, w21 +; CHECK-SD-NEXT: csel w0, w5, w4, ne +; CHECK-SD-NEXT: bl _callee +; CHECK-SD-NEXT: tst w20, w21 +; CHECK-SD-NEXT: csel w0, w0, w19, ne +; CHECK-SD-NEXT: bl _callee +; CHECK-SD-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload +; CHECK-SD-NEXT: ldp x20, x19, [sp, #16] ; 16-byte Folded Reload +; CHECK-SD-NEXT: ldp x22, x21, [sp], #48 ; 16-byte Folded Reload +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: multiccmp: -; GISEL: ; %bb.0: ; %entry -; GISEL-NEXT: stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill -; GISEL-NEXT: stp x29, x30, [sp, #16] ; 16-byte Folded Spill -; GISEL-NEXT: mov x19, x5 -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w20, w8, w9 -; GISEL-NEXT: tst w20, #0x1 -; GISEL-NEXT: csel w0, w5, w4, ne -; GISEL-NEXT: bl _callee -; GISEL-NEXT: tst w20, #0x1 -; GISEL-NEXT: csel w0, w0, w19, ne -; GISEL-NEXT: bl _callee -; GISEL-NEXT: ldp x29, x30, [sp, #16] ; 16-byte Folded Reload -; GISEL-NEXT: ldp x20, x19, [sp], #32 ; 16-byte Folded Reload -; GISEL-NEXT: ret +; CHECK-GI-LABEL: multiccmp: +; CHECK-GI: ; %bb.0: ; %entry +; CHECK-GI-NEXT: stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill +; CHECK-GI-NEXT: stp x29, x30, [sp, #16] ; 16-byte Folded Spill +; CHECK-GI-NEXT: mov x19, x5 +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w20, w8, w9 +; CHECK-GI-NEXT: tst w20, #0x1 +; CHECK-GI-NEXT: csel w0, w5, w4, ne +; CHECK-GI-NEXT: bl _callee +; CHECK-GI-NEXT: tst w20, #0x1 +; CHECK-GI-NEXT: csel w0, w0, w19, ne +; CHECK-GI-NEXT: bl _callee +; CHECK-GI-NEXT: ldp x29, x30, [sp, #16] ; 16-byte Folded Reload +; CHECK-GI-NEXT: ldp x20, x19, [sp], #32 ; 16-byte Folded Reload +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -1100,57 +1100,57 @@ entry: } define i32 @multiccmp2(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %x, i32 %y) #0 { -; SDISEL-LABEL: multiccmp2: -; SDISEL: ; %bb.0: ; %entry -; SDISEL-NEXT: stp x22, x21, [sp, #-48]! ; 16-byte Folded Spill -; SDISEL-NEXT: stp x20, x19, [sp, #16] ; 16-byte Folded Spill -; SDISEL-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill -; SDISEL-NEXT: mov x19, x5 -; SDISEL-NEXT: mov x20, x3 -; SDISEL-NEXT: mov x21, x0 -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: cset w8, gt -; SDISEL-NEXT: cmp w2, w3 -; SDISEL-NEXT: cset w22, ne -; SDISEL-NEXT: tst w8, w22 -; SDISEL-NEXT: csel w0, w5, w4, ne -; SDISEL-NEXT: bl _callee -; SDISEL-NEXT: cmp w21, w20 -; SDISEL-NEXT: cset w8, eq -; SDISEL-NEXT: tst w22, w8 -; SDISEL-NEXT: csel w0, w0, w19, ne -; SDISEL-NEXT: bl _callee -; SDISEL-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload -; SDISEL-NEXT: ldp x20, x19, [sp, #16] ; 16-byte Folded Reload -; SDISEL-NEXT: ldp x22, x21, [sp], #48 ; 16-byte Folded Reload -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: multiccmp2: +; CHECK-SD: ; %bb.0: ; %entry +; CHECK-SD-NEXT: stp x22, x21, [sp, #-48]! ; 16-byte Folded Spill +; CHECK-SD-NEXT: stp x20, x19, [sp, #16] ; 16-byte Folded Spill +; CHECK-SD-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill +; CHECK-SD-NEXT: mov x19, x5 +; CHECK-SD-NEXT: mov x20, x3 +; CHECK-SD-NEXT: mov x21, x0 +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: cset w8, gt +; CHECK-SD-NEXT: cmp w2, w3 +; CHECK-SD-NEXT: cset w22, ne +; CHECK-SD-NEXT: tst w8, w22 +; CHECK-SD-NEXT: csel w0, w5, w4, ne +; CHECK-SD-NEXT: bl _callee +; CHECK-SD-NEXT: cmp w21, w20 +; CHECK-SD-NEXT: cset w8, eq +; CHECK-SD-NEXT: tst w22, w8 +; CHECK-SD-NEXT: csel w0, w0, w19, ne +; CHECK-SD-NEXT: bl _callee +; CHECK-SD-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload +; CHECK-SD-NEXT: ldp x20, x19, [sp, #16] ; 16-byte Folded Reload +; CHECK-SD-NEXT: ldp x22, x21, [sp], #48 ; 16-byte Folded Reload +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: multiccmp2: -; GISEL: ; %bb.0: ; %entry -; GISEL-NEXT: stp x22, x21, [sp, #-48]! ; 16-byte Folded Spill -; GISEL-NEXT: stp x20, x19, [sp, #16] ; 16-byte Folded Spill -; GISEL-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill -; GISEL-NEXT: mov x19, x0 -; GISEL-NEXT: mov x20, x3 -; GISEL-NEXT: mov x21, x5 -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w22, ne -; GISEL-NEXT: and w8, w8, w22 -; GISEL-NEXT: tst w8, #0x1 -; GISEL-NEXT: csel w0, w5, w4, ne -; GISEL-NEXT: bl _callee -; GISEL-NEXT: cmp w19, w20 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: and w8, w22, w8 -; GISEL-NEXT: tst w8, #0x1 -; GISEL-NEXT: csel w0, w0, w21, ne -; GISEL-NEXT: bl _callee -; GISEL-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload -; GISEL-NEXT: ldp x20, x19, [sp, #16] ; 16-byte Folded Reload -; GISEL-NEXT: ldp x22, x21, [sp], #48 ; 16-byte Folded Reload -; GISEL-NEXT: ret +; CHECK-GI-LABEL: multiccmp2: +; CHECK-GI: ; %bb.0: ; %entry +; CHECK-GI-NEXT: stp x22, x21, [sp, #-48]! ; 16-byte Folded Spill +; CHECK-GI-NEXT: stp x20, x19, [sp, #16] ; 16-byte Folded Spill +; CHECK-GI-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill +; CHECK-GI-NEXT: mov x19, x0 +; CHECK-GI-NEXT: mov x20, x3 +; CHECK-GI-NEXT: mov x21, x5 +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w22, ne +; CHECK-GI-NEXT: and w8, w8, w22 +; CHECK-GI-NEXT: tst w8, #0x1 +; CHECK-GI-NEXT: csel w0, w5, w4, ne +; CHECK-GI-NEXT: bl _callee +; CHECK-GI-NEXT: cmp w19, w20 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: and w8, w22, w8 +; CHECK-GI-NEXT: tst w8, #0x1 +; CHECK-GI-NEXT: csel w0, w0, w21, ne +; CHECK-GI-NEXT: bl _callee +; CHECK-GI-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload +; CHECK-GI-NEXT: ldp x20, x19, [sp, #16] ; 16-byte Folded Reload +; CHECK-GI-NEXT: ldp x22, x21, [sp], #48 ; 16-byte Folded Reload +; CHECK-GI-NEXT: ret entry: %c0 = icmp sgt i32 %s0, %s1 %c1 = icmp ne i32 %s2, %s3 @@ -1168,21 +1168,21 @@ entry: declare i32 @callee(i32) define i1 @cmp_and_negative_const(i32 %0, i32 %1) { -; SDISEL-LABEL: cmp_and_negative_const: -; SDISEL: ; %bb.0: -; SDISEL-NEXT: cmn w0, #1 -; SDISEL-NEXT: ccmn w1, #2, #0, eq -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_and_negative_const: +; CHECK-SD: ; %bb.0: +; CHECK-SD-NEXT: cmn w0, #1 +; CHECK-SD-NEXT: ccmn w1, #2, #0, eq +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: cmp_and_negative_const: -; GISEL: ; %bb.0: -; GISEL-NEXT: cmn w0, #1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmn w1, #2 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: cmp_and_negative_const: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: cmn w0, #1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmn w1, #2 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret %3 = icmp eq i32 %0, -1 %4 = icmp eq i32 %1, -2 %5 = and i1 %3, %4 @@ -1190,21 +1190,21 @@ define i1 @cmp_and_negative_const(i32 %0, i32 %1) { } define i1 @cmp_or_negative_const(i32 %a, i32 %b) { -; SDISEL-LABEL: cmp_or_negative_const: -; SDISEL: ; %bb.0: -; SDISEL-NEXT: cmn w0, #1 -; SDISEL-NEXT: ccmn w1, #2, #4, ne -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_or_negative_const: +; CHECK-SD: ; %bb.0: +; CHECK-SD-NEXT: cmn w0, #1 +; CHECK-SD-NEXT: ccmn w1, #2, #4, ne +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: cmp_or_negative_const: -; GISEL: ; %bb.0: -; GISEL-NEXT: cmn w0, #1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: cmn w1, #2 -; GISEL-NEXT: cset w9, eq -; GISEL-NEXT: orr w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: cmp_or_negative_const: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: cmn w0, #1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: cmn w1, #2 +; CHECK-GI-NEXT: cset w9, eq +; CHECK-GI-NEXT: orr w0, w8, w9 +; CHECK-GI-NEXT: ret %cmp = icmp eq i32 %a, -1 %cmp1 = icmp eq i32 %b, -2 %or.cond = or i1 %cmp, %cmp1 diff --git a/llvm/test/CodeGen/AArch64/cmp-chains.ll b/llvm/test/CodeGen/AArch64/cmp-chains.ll index 4b816df..3620444 100644 --- a/llvm/test/CodeGen/AArch64/cmp-chains.ll +++ b/llvm/test/CodeGen/AArch64/cmp-chains.ll @@ -1,26 +1,26 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK,SDISEL -; RUN: llc < %s -mtriple=aarch64-- -global-isel | FileCheck %s --check-prefixes=CHECK,GISEL +; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc < %s -mtriple=aarch64-- -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI ; Ensure chains of comparisons produce chains of `ccmp` ; (x0 < x1) && (x2 > x3) define i32 @cmp_and2(i32 %0, i32 %1, i32 %2, i32 %3) { -; SDISEL-LABEL: cmp_and2: -; SDISEL: // %bb.0: -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lo -; SDISEL-NEXT: cset w0, hi -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_and2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lo +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: cmp_and2: -; GISEL: // %bb.0: -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: cmp_and2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret %5 = icmp ult i32 %0, %1 %6 = icmp ugt i32 %2, %3 %7 = select i1 %5, i1 %6, i1 false @@ -30,25 +30,25 @@ define i32 @cmp_and2(i32 %0, i32 %1, i32 %2, i32 %3) { ; (x0 < x1) && (x2 > x3) && (x4 != x5) define i32 @cmp_and3(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5) { -; SDISEL-LABEL: cmp_and3: -; SDISEL: // %bb.0: -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, lo -; SDISEL-NEXT: ccmp w4, w5, #4, hi -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_and3: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, lo +; CHECK-SD-NEXT: ccmp w4, w5, #4, hi +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: cmp_and3: -; GISEL: // %bb.0: -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: cmp_and3: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: and w8, w8, w9 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret %7 = icmp ult i32 %0, %1 %8 = icmp ugt i32 %2, %3 %9 = select i1 %7, i1 %8, i1 false @@ -60,29 +60,29 @@ define i32 @cmp_and3(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5) { ; (x0 < x1) && (x2 > x3) && (x4 != x5) && (x6 == x7) define i32 @cmp_and4(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7) { -; SDISEL-LABEL: cmp_and4: -; SDISEL: // %bb.0: -; SDISEL-NEXT: cmp w2, w3 -; SDISEL-NEXT: ccmp w0, w1, #2, hi -; SDISEL-NEXT: ccmp w4, w5, #4, lo -; SDISEL-NEXT: ccmp w6, w7, #0, ne -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_and4: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmp w2, w3 +; CHECK-SD-NEXT: ccmp w0, w1, #2, hi +; CHECK-SD-NEXT: ccmp w4, w5, #4, lo +; CHECK-SD-NEXT: ccmp w6, w7, #0, ne +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: cmp_and4: -; GISEL: // %bb.0: -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w8, hi -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: cset w10, ne -; GISEL-NEXT: cmp w6, w7 -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: cset w11, eq -; GISEL-NEXT: and w9, w10, w11 -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: cmp_and4: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w8, hi +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: cset w10, ne +; CHECK-GI-NEXT: cmp w6, w7 +; CHECK-GI-NEXT: and w8, w8, w9 +; CHECK-GI-NEXT: cset w11, eq +; CHECK-GI-NEXT: and w9, w10, w11 +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret %9 = icmp ugt i32 %2, %3 %10 = icmp ult i32 %0, %1 %11 = select i1 %9, i1 %10, i1 false @@ -96,22 +96,22 @@ define i32 @cmp_and4(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 ; (x0 < x1) || (x2 > x3) define i32 @cmp_or2(i32 %0, i32 %1, i32 %2, i32 %3) { -; SDISEL-LABEL: cmp_or2: -; SDISEL: // %bb.0: -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #0, hs -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_or2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #0, hs +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: cmp_or2: -; GISEL: // %bb.0: -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: and w0, w8, #0x1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: cmp_or2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: orr w8, w8, w9 +; CHECK-GI-NEXT: and w0, w8, #0x1 +; CHECK-GI-NEXT: ret %5 = icmp ult i32 %0, %1 %6 = icmp ne i32 %2, %3 %7 = select i1 %5, i1 true, i1 %6 @@ -121,26 +121,26 @@ define i32 @cmp_or2(i32 %0, i32 %1, i32 %2, i32 %3) { ; (x0 < x1) || (x2 > x3) || (x4 != x5) define i32 @cmp_or3(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5) { -; SDISEL-LABEL: cmp_or3: -; SDISEL: // %bb.0: -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, hs -; SDISEL-NEXT: ccmp w4, w5, #0, ls -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_or3: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, hs +; CHECK-SD-NEXT: ccmp w4, w5, #0, ls +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: cmp_or3: -; GISEL: // %bb.0: -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: and w0, w8, #0x1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: cmp_or3: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: orr w8, w8, w9 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: orr w8, w8, w9 +; CHECK-GI-NEXT: and w0, w8, #0x1 +; CHECK-GI-NEXT: ret %7 = icmp ult i32 %0, %1 %8 = icmp ugt i32 %2, %3 %9 = select i1 %7, i1 true, i1 %8 @@ -152,30 +152,30 @@ define i32 @cmp_or3(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5) { ; (x0 < x1) || (x2 > x3) || (x4 != x5) || (x6 == x7) define i32 @cmp_or4(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7) { -; SDISEL-LABEL: cmp_or4: -; SDISEL: // %bb.0: -; SDISEL-NEXT: cmp w0, w1 -; SDISEL-NEXT: ccmp w2, w3, #2, hs -; SDISEL-NEXT: ccmp w4, w5, #0, ls -; SDISEL-NEXT: ccmp w6, w7, #4, eq -; SDISEL-NEXT: cset w0, eq -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: cmp_or4: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmp w0, w1 +; CHECK-SD-NEXT: ccmp w2, w3, #2, hs +; CHECK-SD-NEXT: ccmp w4, w5, #0, ls +; CHECK-SD-NEXT: ccmp w6, w7, #4, eq +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: cmp_or4: -; GISEL: // %bb.0: -; GISEL-NEXT: cmp w0, w1 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w3 -; GISEL-NEXT: cset w9, hi -; GISEL-NEXT: cmp w4, w5 -; GISEL-NEXT: cset w10, ne -; GISEL-NEXT: cmp w6, w7 -; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: cset w11, eq -; GISEL-NEXT: orr w9, w10, w11 -; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: and w0, w8, #0x1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: cmp_or4: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w0, w1 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w3 +; CHECK-GI-NEXT: cset w9, hi +; CHECK-GI-NEXT: cmp w4, w5 +; CHECK-GI-NEXT: cset w10, ne +; CHECK-GI-NEXT: cmp w6, w7 +; CHECK-GI-NEXT: orr w8, w8, w9 +; CHECK-GI-NEXT: cset w11, eq +; CHECK-GI-NEXT: orr w9, w10, w11 +; CHECK-GI-NEXT: orr w8, w8, w9 +; CHECK-GI-NEXT: and w0, w8, #0x1 +; CHECK-GI-NEXT: ret %9 = icmp ult i32 %0, %1 %10 = icmp ugt i32 %2, %3 %11 = select i1 %9, i1 true, i1 %10 @@ -189,22 +189,22 @@ define i32 @cmp_or4(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 ; (x0 != 0) || (x1 != 0) define i32 @true_or2(i32 %0, i32 %1) { -; SDISEL-LABEL: true_or2: -; SDISEL: // %bb.0: -; SDISEL-NEXT: orr w8, w0, w1 -; SDISEL-NEXT: cmp w8, #0 -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: true_or2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: orr w8, w0, w1 +; CHECK-SD-NEXT: cmp w8, #0 +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: true_or2: -; GISEL: // %bb.0: -; GISEL-NEXT: cmp w0, #0 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w1, #0 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: and w0, w8, #0x1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: true_or2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w0, #0 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w1, #0 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: orr w8, w8, w9 +; CHECK-GI-NEXT: and w0, w8, #0x1 +; CHECK-GI-NEXT: ret %3 = icmp ne i32 %0, 0 %4 = icmp ne i32 %1, 0 %5 = select i1 %3, i1 true, i1 %4 @@ -214,26 +214,26 @@ define i32 @true_or2(i32 %0, i32 %1) { ; (x0 != 0) || (x1 != 0) || (x2 != 0) define i32 @true_or3(i32 %0, i32 %1, i32 %2) { -; SDISEL-LABEL: true_or3: -; SDISEL: // %bb.0: -; SDISEL-NEXT: orr w8, w0, w1 -; SDISEL-NEXT: orr w8, w8, w2 -; SDISEL-NEXT: cmp w8, #0 -; SDISEL-NEXT: cset w0, ne -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: true_or3: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: orr w8, w0, w1 +; CHECK-SD-NEXT: orr w8, w8, w2 +; CHECK-SD-NEXT: cmp w8, #0 +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: true_or3: -; GISEL: // %bb.0: -; GISEL-NEXT: cmp w0, #0 -; GISEL-NEXT: cset w8, ne -; GISEL-NEXT: cmp w1, #0 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: cmp w2, #0 -; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: cset w9, ne -; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: and w0, w8, #0x1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: true_or3: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w0, #0 +; CHECK-GI-NEXT: cset w8, ne +; CHECK-GI-NEXT: cmp w1, #0 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: cmp w2, #0 +; CHECK-GI-NEXT: orr w8, w8, w9 +; CHECK-GI-NEXT: cset w9, ne +; CHECK-GI-NEXT: orr w8, w8, w9 +; CHECK-GI-NEXT: and w0, w8, #0x1 +; CHECK-GI-NEXT: ret %4 = icmp ne i32 %0, 0 %5 = icmp ne i32 %1, 0 %6 = select i1 %4, i1 true, i1 %5 @@ -260,22 +260,22 @@ define i32 @neg_range_int(i32 %a, i32 %b, i32 %c) { ; (b > -(d | 1) && a < c) define i32 @neg_range_int_comp(i32 %a, i32 %b, i32 %c, i32 %d) { -; SDISEL-LABEL: neg_range_int_comp: -; SDISEL: // %bb.0: -; SDISEL-NEXT: orr w8, w3, #0x1 -; SDISEL-NEXT: cmp w0, w2 -; SDISEL-NEXT: ccmn w1, w8, #4, lt -; SDISEL-NEXT: csel w0, w1, w0, gt -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: neg_range_int_comp: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: orr w8, w3, #0x1 +; CHECK-SD-NEXT: cmp w0, w2 +; CHECK-SD-NEXT: ccmn w1, w8, #4, lt +; CHECK-SD-NEXT: csel w0, w1, w0, gt +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: neg_range_int_comp: -; GISEL: // %bb.0: -; GISEL-NEXT: orr w8, w3, #0x1 -; GISEL-NEXT: cmp w0, w2 -; GISEL-NEXT: neg w8, w8 -; GISEL-NEXT: ccmp w1, w8, #4, lt -; GISEL-NEXT: csel w0, w1, w0, gt -; GISEL-NEXT: ret +; CHECK-GI-LABEL: neg_range_int_comp: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: orr w8, w3, #0x1 +; CHECK-GI-NEXT: cmp w0, w2 +; CHECK-GI-NEXT: neg w8, w8 +; CHECK-GI-NEXT: ccmp w1, w8, #4, lt +; CHECK-GI-NEXT: csel w0, w1, w0, gt +; CHECK-GI-NEXT: ret %dor = or i32 %d, 1 %negd = sub i32 0, %dor %cmp = icmp sgt i32 %b, %negd @@ -287,22 +287,22 @@ define i32 @neg_range_int_comp(i32 %a, i32 %b, i32 %c, i32 %d) { ; (b >u -(d | 1) && a < c) define i32 @neg_range_int_comp_u(i32 %a, i32 %b, i32 %c, i32 %d) { -; SDISEL-LABEL: neg_range_int_comp_u: -; SDISEL: // %bb.0: -; SDISEL-NEXT: orr w8, w3, #0x1 -; SDISEL-NEXT: cmp w0, w2 -; SDISEL-NEXT: ccmn w1, w8, #0, lt -; SDISEL-NEXT: csel w0, w1, w0, hi -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: neg_range_int_comp_u: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: orr w8, w3, #0x1 +; CHECK-SD-NEXT: cmp w0, w2 +; CHECK-SD-NEXT: ccmn w1, w8, #0, lt +; CHECK-SD-NEXT: csel w0, w1, w0, hi +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: neg_range_int_comp_u: -; GISEL: // %bb.0: -; GISEL-NEXT: orr w8, w3, #0x1 -; GISEL-NEXT: cmp w0, w2 -; GISEL-NEXT: neg w8, w8 -; GISEL-NEXT: ccmp w1, w8, #0, lt -; GISEL-NEXT: csel w0, w1, w0, hi -; GISEL-NEXT: ret +; CHECK-GI-LABEL: neg_range_int_comp_u: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: orr w8, w3, #0x1 +; CHECK-GI-NEXT: cmp w0, w2 +; CHECK-GI-NEXT: neg w8, w8 +; CHECK-GI-NEXT: ccmp w1, w8, #0, lt +; CHECK-GI-NEXT: csel w0, w1, w0, hi +; CHECK-GI-NEXT: ret %dor = or i32 %d, 1 %negd = sub i32 0, %dor %cmp = icmp ugt i32 %b, %negd @@ -314,22 +314,22 @@ define i32 @neg_range_int_comp_u(i32 %a, i32 %b, i32 %c, i32 %d) { ; (b > -(d | 1) && a u < c) define i32 @neg_range_int_comp_ua(i32 %a, i32 %b, i32 %c, i32 %d) { -; SDISEL-LABEL: neg_range_int_comp_ua: -; SDISEL: // %bb.0: -; SDISEL-NEXT: orr w8, w3, #0x1 -; SDISEL-NEXT: cmp w0, w2 -; SDISEL-NEXT: ccmn w1, w8, #4, lo -; SDISEL-NEXT: csel w0, w1, w0, gt -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: neg_range_int_comp_ua: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: orr w8, w3, #0x1 +; CHECK-SD-NEXT: cmp w0, w2 +; CHECK-SD-NEXT: ccmn w1, w8, #4, lo +; CHECK-SD-NEXT: csel w0, w1, w0, gt +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: neg_range_int_comp_ua: -; GISEL: // %bb.0: -; GISEL-NEXT: orr w8, w3, #0x1 -; GISEL-NEXT: cmp w0, w2 -; GISEL-NEXT: neg w8, w8 -; GISEL-NEXT: ccmp w1, w8, #4, lo -; GISEL-NEXT: csel w0, w1, w0, gt -; GISEL-NEXT: ret +; CHECK-GI-LABEL: neg_range_int_comp_ua: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: orr w8, w3, #0x1 +; CHECK-GI-NEXT: cmp w0, w2 +; CHECK-GI-NEXT: neg w8, w8 +; CHECK-GI-NEXT: ccmp w1, w8, #4, lo +; CHECK-GI-NEXT: csel w0, w1, w0, gt +; CHECK-GI-NEXT: ret %dor = or i32 %d, 1 %negd = sub i32 0, %dor %cmp = icmp sgt i32 %b, %negd @@ -341,19 +341,19 @@ define i32 @neg_range_int_comp_ua(i32 %a, i32 %b, i32 %c, i32 %d) { ; (b <= -3 && a > c) define i32 @neg_range_int_2(i32 %a, i32 %b, i32 %c) { -; SDISEL-LABEL: neg_range_int_2: -; SDISEL: // %bb.0: -; SDISEL-NEXT: cmp w0, w2 -; SDISEL-NEXT: ccmn w1, #4, #4, gt -; SDISEL-NEXT: csel w0, w1, w0, gt -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: neg_range_int_2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmp w0, w2 +; CHECK-SD-NEXT: ccmn w1, #4, #4, gt +; CHECK-SD-NEXT: csel w0, w1, w0, gt +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: neg_range_int_2: -; GISEL: // %bb.0: -; GISEL-NEXT: cmp w0, w2 -; GISEL-NEXT: ccmn w1, #3, #8, gt -; GISEL-NEXT: csel w0, w1, w0, ge -; GISEL-NEXT: ret +; CHECK-GI-LABEL: neg_range_int_2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w0, w2 +; CHECK-GI-NEXT: ccmn w1, #3, #8, gt +; CHECK-GI-NEXT: csel w0, w1, w0, ge +; CHECK-GI-NEXT: ret %cmp = icmp sge i32 %b, -3 %cmp1 = icmp sgt i32 %a, %c %or.cond = and i1 %cmp, %cmp1 @@ -363,22 +363,22 @@ define i32 @neg_range_int_2(i32 %a, i32 %b, i32 %c) { ; (b < -(d | 1) && a >= c) define i32 @neg_range_int_comp2(i32 %a, i32 %b, i32 %c, i32 %d) { -; SDISEL-LABEL: neg_range_int_comp2: -; SDISEL: // %bb.0: -; SDISEL-NEXT: orr w8, w3, #0x1 -; SDISEL-NEXT: cmp w0, w2 -; SDISEL-NEXT: ccmn w1, w8, #0, ge -; SDISEL-NEXT: csel w0, w1, w0, lt -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: neg_range_int_comp2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: orr w8, w3, #0x1 +; CHECK-SD-NEXT: cmp w0, w2 +; CHECK-SD-NEXT: ccmn w1, w8, #0, ge +; CHECK-SD-NEXT: csel w0, w1, w0, lt +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: neg_range_int_comp2: -; GISEL: // %bb.0: -; GISEL-NEXT: orr w8, w3, #0x1 -; GISEL-NEXT: cmp w0, w2 -; GISEL-NEXT: neg w8, w8 -; GISEL-NEXT: ccmp w1, w8, #0, ge -; GISEL-NEXT: csel w0, w1, w0, lt -; GISEL-NEXT: ret +; CHECK-GI-LABEL: neg_range_int_comp2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: orr w8, w3, #0x1 +; CHECK-GI-NEXT: cmp w0, w2 +; CHECK-GI-NEXT: neg w8, w8 +; CHECK-GI-NEXT: ccmp w1, w8, #0, ge +; CHECK-GI-NEXT: csel w0, w1, w0, lt +; CHECK-GI-NEXT: ret %dor = or i32 %d, 1 %negd = sub i32 0, %dor %cmp = icmp slt i32 %b, %negd @@ -390,22 +390,22 @@ define i32 @neg_range_int_comp2(i32 %a, i32 %b, i32 %c, i32 %d) { ; (b <u -(d | 1) && a > c) define i32 @neg_range_int_comp_u2(i32 %a, i32 %b, i32 %c, i32 %d) { -; SDISEL-LABEL: neg_range_int_comp_u2: -; SDISEL: // %bb.0: -; SDISEL-NEXT: orr w8, w3, #0x1 -; SDISEL-NEXT: cmp w0, w2 -; SDISEL-NEXT: ccmn w1, w8, #2, gt -; SDISEL-NEXT: csel w0, w1, w0, lo -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: neg_range_int_comp_u2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: orr w8, w3, #0x1 +; CHECK-SD-NEXT: cmp w0, w2 +; CHECK-SD-NEXT: ccmn w1, w8, #2, gt +; CHECK-SD-NEXT: csel w0, w1, w0, lo +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: neg_range_int_comp_u2: -; GISEL: // %bb.0: -; GISEL-NEXT: orr w8, w3, #0x1 -; GISEL-NEXT: cmp w0, w2 -; GISEL-NEXT: neg w8, w8 -; GISEL-NEXT: ccmp w1, w8, #2, gt -; GISEL-NEXT: csel w0, w1, w0, lo -; GISEL-NEXT: ret +; CHECK-GI-LABEL: neg_range_int_comp_u2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: orr w8, w3, #0x1 +; CHECK-GI-NEXT: cmp w0, w2 +; CHECK-GI-NEXT: neg w8, w8 +; CHECK-GI-NEXT: ccmp w1, w8, #2, gt +; CHECK-GI-NEXT: csel w0, w1, w0, lo +; CHECK-GI-NEXT: ret %dor = or i32 %d, 1 %negd = sub i32 0, %dor %cmp = icmp ult i32 %b, %negd @@ -417,22 +417,22 @@ define i32 @neg_range_int_comp_u2(i32 %a, i32 %b, i32 %c, i32 %d) { ; (b > -(d | 1) && a u > c) define i32 @neg_range_int_comp_ua2(i32 %a, i32 %b, i32 %c, i32 %d) { -; SDISEL-LABEL: neg_range_int_comp_ua2: -; SDISEL: // %bb.0: -; SDISEL-NEXT: orr w8, w3, #0x1 -; SDISEL-NEXT: cmp w0, w2 -; SDISEL-NEXT: ccmn w1, w8, #4, hi -; SDISEL-NEXT: csel w0, w1, w0, gt -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: neg_range_int_comp_ua2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: orr w8, w3, #0x1 +; CHECK-SD-NEXT: cmp w0, w2 +; CHECK-SD-NEXT: ccmn w1, w8, #4, hi +; CHECK-SD-NEXT: csel w0, w1, w0, gt +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: neg_range_int_comp_ua2: -; GISEL: // %bb.0: -; GISEL-NEXT: orr w8, w3, #0x1 -; GISEL-NEXT: cmp w0, w2 -; GISEL-NEXT: neg w8, w8 -; GISEL-NEXT: ccmp w1, w8, #4, hi -; GISEL-NEXT: csel w0, w1, w0, gt -; GISEL-NEXT: ret +; CHECK-GI-LABEL: neg_range_int_comp_ua2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: orr w8, w3, #0x1 +; CHECK-GI-NEXT: cmp w0, w2 +; CHECK-GI-NEXT: neg w8, w8 +; CHECK-GI-NEXT: ccmp w1, w8, #4, hi +; CHECK-GI-NEXT: csel w0, w1, w0, gt +; CHECK-GI-NEXT: ret %dor = or i32 %d, 1 %negd = sub i32 0, %dor %cmp = icmp sgt i32 %b, %negd @@ -444,22 +444,22 @@ define i32 @neg_range_int_comp_ua2(i32 %a, i32 %b, i32 %c, i32 %d) { ; (b > -(d | 1) && a u == c) define i32 @neg_range_int_comp_ua3(i32 %a, i32 %b, i32 %c, i32 %d) { -; SDISEL-LABEL: neg_range_int_comp_ua3: -; SDISEL: // %bb.0: -; SDISEL-NEXT: orr w8, w3, #0x1 -; SDISEL-NEXT: cmp w0, w2 -; SDISEL-NEXT: ccmn w1, w8, #4, eq -; SDISEL-NEXT: csel w0, w1, w0, gt -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: neg_range_int_comp_ua3: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: orr w8, w3, #0x1 +; CHECK-SD-NEXT: cmp w0, w2 +; CHECK-SD-NEXT: ccmn w1, w8, #4, eq +; CHECK-SD-NEXT: csel w0, w1, w0, gt +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: neg_range_int_comp_ua3: -; GISEL: // %bb.0: -; GISEL-NEXT: orr w8, w3, #0x1 -; GISEL-NEXT: cmp w0, w2 -; GISEL-NEXT: neg w8, w8 -; GISEL-NEXT: ccmp w1, w8, #4, eq -; GISEL-NEXT: csel w0, w1, w0, gt -; GISEL-NEXT: ret +; CHECK-GI-LABEL: neg_range_int_comp_ua3: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: orr w8, w3, #0x1 +; CHECK-GI-NEXT: cmp w0, w2 +; CHECK-GI-NEXT: neg w8, w8 +; CHECK-GI-NEXT: ccmp w1, w8, #4, eq +; CHECK-GI-NEXT: csel w0, w1, w0, gt +; CHECK-GI-NEXT: ret %dor = or i32 %d, 1 %negd = sub i32 0, %dor %cmp = icmp sgt i32 %b, %negd @@ -471,26 +471,26 @@ define i32 @neg_range_int_comp_ua3(i32 %a, i32 %b, i32 %c, i32 %d) { ; -(a | 1) > (b | 3) && a < c define i32 @neg_range_int_c(i32 %a, i32 %b, i32 %c) { -; SDISEL-LABEL: neg_range_int_c: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: orr w8, w0, #0x1 -; SDISEL-NEXT: orr w9, w1, #0x3 -; SDISEL-NEXT: cmn w9, w8 -; SDISEL-NEXT: ccmp w2, w0, #2, lo -; SDISEL-NEXT: cset w0, lo -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: neg_range_int_c: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: orr w8, w0, #0x1 +; CHECK-SD-NEXT: orr w9, w1, #0x3 +; CHECK-SD-NEXT: cmn w9, w8 +; CHECK-SD-NEXT: ccmp w2, w0, #2, lo +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: neg_range_int_c: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: orr w8, w0, #0x1 -; GISEL-NEXT: orr w9, w1, #0x3 -; GISEL-NEXT: neg w8, w8 -; GISEL-NEXT: cmp w9, w8 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: cmp w2, w0 -; GISEL-NEXT: cset w9, lo -; GISEL-NEXT: and w0, w8, w9 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: neg_range_int_c: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: orr w8, w0, #0x1 +; CHECK-GI-NEXT: orr w9, w1, #0x3 +; CHECK-GI-NEXT: neg w8, w8 +; CHECK-GI-NEXT: cmp w9, w8 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: cmp w2, w0 +; CHECK-GI-NEXT: cset w9, lo +; CHECK-GI-NEXT: and w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %or = or i32 %a, 1 %sub = sub i32 0, %or diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll index 13434fa..7686740 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll @@ -203,93 +203,89 @@ define <12 x float> @abp90c12(<12 x float> %a, <12 x float> %b, <12 x float> %c) ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1 ; CHECK-NEXT: // kill: def $s3 killed $s3 def $q3 +; CHECK-NEXT: ldr s17, [sp, #40] +; CHECK-NEXT: add x10, sp, #56 ; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0 -; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2 -; CHECK-NEXT: ldr s17, [sp, #32] -; CHECK-NEXT: // kill: def $s5 killed $s5 def $q5 ; CHECK-NEXT: add x9, sp, #48 -; CHECK-NEXT: add x10, sp, #64 ; CHECK-NEXT: mov v1.s[1], v3.s[0] +; CHECK-NEXT: ldr s3, [sp, #32] +; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2 ; CHECK-NEXT: mov v0.s[1], v2.s[0] +; CHECK-NEXT: ld1 { v17.s }[1], [x10] +; CHECK-NEXT: // kill: def $s5 killed $s5 def $q5 +; CHECK-NEXT: ldr s16, [sp, #8] ; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4 -; CHECK-NEXT: add x11, sp, #72 -; CHECK-NEXT: ld1 { v17.s }[1], [x9] -; CHECK-NEXT: ldr s18, [x10] -; CHECK-NEXT: add x9, sp, #80 -; CHECK-NEXT: add x10, sp, #56 -; CHECK-NEXT: // kill: def $s6 killed $s6 def $q6 +; CHECK-NEXT: add x10, sp, #24 +; CHECK-NEXT: ld1 { v3.s }[1], [x9] +; CHECK-NEXT: add x9, sp, #72 ; CHECK-NEXT: // kill: def $s7 killed $s7 def $q7 -; CHECK-NEXT: ldr s16, [sp, #8] -; CHECK-NEXT: ldr s3, [sp, #96] -; CHECK-NEXT: ld1 { v18.s }[1], [x9] -; CHECK-NEXT: add x9, sp, #88 +; CHECK-NEXT: // kill: def $s6 killed $s6 def $q6 ; CHECK-NEXT: ldr s2, [sp] +; CHECK-NEXT: ld1 { v16.s }[1], [x10] +; CHECK-NEXT: add x10, sp, #112 +; CHECK-NEXT: ldr s20, [sp, #136] ; CHECK-NEXT: mov v1.s[2], v5.s[0] -; CHECK-NEXT: ldr s5, [sp, #40] +; CHECK-NEXT: ld1 { v17.s }[2], [x9] +; CHECK-NEXT: add x9, sp, #64 +; CHECK-NEXT: ldr s5, [sp, #96] +; CHECK-NEXT: ld1 { v3.s }[2], [x9] ; CHECK-NEXT: mov v0.s[2], v4.s[0] +; CHECK-NEXT: add x9, sp, #88 +; CHECK-NEXT: ldr s4, [sp, #104] +; CHECK-NEXT: ldr s19, [sp, #192] ; CHECK-NEXT: ld1 { v5.s }[1], [x10] -; CHECK-NEXT: ldr s19, [x11] +; CHECK-NEXT: add x10, sp, #80 +; CHECK-NEXT: ld1 { v17.s }[3], [x9] +; CHECK-NEXT: mov v1.s[3], v7.s[0] +; CHECK-NEXT: add x9, sp, #120 +; CHECK-NEXT: ld1 { v3.s }[3], [x10] +; CHECK-NEXT: ld1 { v4.s }[1], [x9] +; CHECK-NEXT: ldr s7, [sp, #128] ; CHECK-NEXT: add x10, sp, #144 -; CHECK-NEXT: zip1 v4.2d, v17.2d, v18.2d -; CHECK-NEXT: add x11, sp, #160 -; CHECK-NEXT: ldr s18, [sp, #136] -; CHECK-NEXT: ld1 { v19.s }[1], [x9] ; CHECK-NEXT: mov v0.s[3], v6.s[0] -; CHECK-NEXT: ldr s6, [sp, #128] -; CHECK-NEXT: mov v1.s[3], v7.s[0] -; CHECK-NEXT: add x9, sp, #24 -; CHECK-NEXT: ldr s7, [sp, #104] -; CHECK-NEXT: ld1 { v16.s }[1], [x9] -; CHECK-NEXT: add x9, sp, #112 -; CHECK-NEXT: ld1 { v6.s }[1], [x10] -; CHECK-NEXT: zip1 v5.2d, v5.2d, v19.2d -; CHECK-NEXT: add x10, sp, #120 -; CHECK-NEXT: ld1 { v3.s }[1], [x9] +; CHECK-NEXT: add x9, sp, #16 ; CHECK-NEXT: ld1 { v7.s }[1], [x10] -; CHECK-NEXT: ldr s17, [x11] -; CHECK-NEXT: add x9, sp, #176 -; CHECK-NEXT: add x10, sp, #16 -; CHECK-NEXT: add x11, sp, #168 -; CHECK-NEXT: ld1 { v17.s }[1], [x9] -; CHECK-NEXT: ld1 { v2.s }[1], [x10] -; CHECK-NEXT: add x9, sp, #152 -; CHECK-NEXT: fmul v19.4s, v5.4s, v1.4s -; CHECK-NEXT: fmul v20.4s, v7.4s, v16.4s -; CHECK-NEXT: fmul v16.4s, v3.4s, v16.4s -; CHECK-NEXT: fmul v1.4s, v4.4s, v1.4s -; CHECK-NEXT: ld1 { v18.s }[1], [x9] -; CHECK-NEXT: ldr s21, [x11] -; CHECK-NEXT: zip1 v6.2d, v6.2d, v17.2d -; CHECK-NEXT: ldr s17, [sp, #192] -; CHECK-NEXT: add x9, sp, #184 +; CHECK-NEXT: ld1 { v2.s }[1], [x9] +; CHECK-NEXT: add x9, sp, #160 +; CHECK-NEXT: fmul v6.4s, v17.4s, v1.4s +; CHECK-NEXT: fmul v18.4s, v4.4s, v16.4s +; CHECK-NEXT: fmul v16.4s, v5.4s, v16.4s +; CHECK-NEXT: fmul v1.4s, v3.4s, v1.4s ; CHECK-NEXT: add x10, sp, #208 -; CHECK-NEXT: ld1 { v21.s }[1], [x9] +; CHECK-NEXT: ld1 { v7.s }[2], [x9] +; CHECK-NEXT: add x9, sp, #152 +; CHECK-NEXT: ld1 { v19.s }[1], [x10] +; CHECK-NEXT: ld1 { v20.s }[1], [x9] +; CHECK-NEXT: add x9, sp, #176 +; CHECK-NEXT: add x10, sp, #184 +; CHECK-NEXT: fneg v6.4s, v6.4s +; CHECK-NEXT: fneg v18.4s, v18.4s +; CHECK-NEXT: fmla v16.4s, v2.4s, v4.4s +; CHECK-NEXT: fmla v1.4s, v0.4s, v17.4s +; CHECK-NEXT: ld1 { v7.s }[3], [x9] +; CHECK-NEXT: add x9, sp, #168 +; CHECK-NEXT: ld1 { v20.s }[2], [x9] +; CHECK-NEXT: ldr s4, [sp, #200] ; CHECK-NEXT: add x9, sp, #216 -; CHECK-NEXT: fneg v19.4s, v19.4s -; CHECK-NEXT: fneg v20.4s, v20.4s -; CHECK-NEXT: fmla v16.4s, v2.4s, v7.4s -; CHECK-NEXT: fmla v1.4s, v0.4s, v5.4s -; CHECK-NEXT: ld1 { v17.s }[1], [x10] -; CHECK-NEXT: ldr s5, [sp, #200] -; CHECK-NEXT: zip1 v7.2d, v18.2d, v21.2d -; CHECK-NEXT: ld1 { v5.s }[1], [x9] -; CHECK-NEXT: fmla v19.4s, v0.4s, v4.4s -; CHECK-NEXT: fmla v20.4s, v2.4s, v3.4s -; CHECK-NEXT: fsub v0.4s, v6.4s, v1.4s -; CHECK-NEXT: fsub v1.4s, v17.4s, v16.4s -; CHECK-NEXT: fadd v2.4s, v7.4s, v19.4s -; CHECK-NEXT: fadd v3.4s, v5.4s, v20.4s +; CHECK-NEXT: fmla v6.4s, v0.4s, v3.4s +; CHECK-NEXT: fmla v18.4s, v2.4s, v5.4s +; CHECK-NEXT: ld1 { v4.s }[1], [x9] +; CHECK-NEXT: fsub v0.4s, v7.4s, v1.4s +; CHECK-NEXT: fsub v1.4s, v19.4s, v16.4s +; CHECK-NEXT: ld1 { v20.s }[3], [x10] +; CHECK-NEXT: fadd v2.4s, v4.4s, v18.4s +; CHECK-NEXT: fadd v3.4s, v20.4s, v6.4s ; CHECK-NEXT: ext v4.16b, v0.16b, v1.16b, #12 -; CHECK-NEXT: ext v5.16b, v2.16b, v3.16b, #12 -; CHECK-NEXT: trn2 v1.4s, v1.4s, v3.4s +; CHECK-NEXT: ext v5.16b, v3.16b, v2.16b, #12 +; CHECK-NEXT: trn2 v1.4s, v1.4s, v2.4s ; CHECK-NEXT: ext v4.16b, v0.16b, v4.16b, #12 -; CHECK-NEXT: ext v5.16b, v2.16b, v5.16b, #8 +; CHECK-NEXT: ext v5.16b, v3.16b, v5.16b, #8 ; CHECK-NEXT: rev64 v4.4s, v4.4s -; CHECK-NEXT: trn2 v3.4s, v4.4s, v5.4s -; CHECK-NEXT: zip2 v4.4s, v0.4s, v2.4s -; CHECK-NEXT: zip1 v0.4s, v0.4s, v2.4s -; CHECK-NEXT: ext v1.16b, v3.16b, v1.16b, #8 -; CHECK-NEXT: mov v4.d[1], v3.d[0] +; CHECK-NEXT: trn2 v2.4s, v4.4s, v5.4s +; CHECK-NEXT: zip2 v4.4s, v0.4s, v3.4s +; CHECK-NEXT: zip1 v0.4s, v0.4s, v3.4s +; CHECK-NEXT: ext v1.16b, v2.16b, v1.16b, #8 +; CHECK-NEXT: mov v4.d[1], v2.d[0] ; CHECK-NEXT: str q0, [x8] ; CHECK-NEXT: stp q4, q1, [x8, #16] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/concat-vector.ll b/llvm/test/CodeGen/AArch64/concat-vector.ll index e6f27b9..acf15f1 100644 --- a/llvm/test/CodeGen/AArch64/concat-vector.ll +++ b/llvm/test/CodeGen/AArch64/concat-vector.ll @@ -186,9 +186,8 @@ define <16 x i8> @concat_v16s8_v4s8_load(ptr %ptrA, ptr %ptrB, ptr %ptrC, ptr %p ; CHECK: // %bb.0: ; CHECK-NEXT: ldr s0, [x0] ; CHECK-NEXT: ld1 { v0.s }[1], [x1] -; CHECK-NEXT: ldr s1, [x2] -; CHECK-NEXT: ld1 { v1.s }[1], [x3] -; CHECK-NEXT: zip1 v0.2d, v0.2d, v1.2d +; CHECK-NEXT: ld1 { v0.s }[2], [x2] +; CHECK-NEXT: ld1 { v0.s }[3], [x3] ; CHECK-NEXT: ret %A = load <4 x i8>, ptr %ptrA %B = load <4 x i8>, ptr %ptrB diff --git a/llvm/test/CodeGen/AArch64/dag-combine-select.ll b/llvm/test/CodeGen/AArch64/dag-combine-select.ll index 56208f1..02b0077 100644 --- a/llvm/test/CodeGen/AArch64/dag-combine-select.ll +++ b/llvm/test/CodeGen/AArch64/dag-combine-select.ll @@ -1,26 +1,26 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple arm64-none-eabi -o - %s | FileCheck %s --check-prefixes=CHECK,SDISEL -; RUN: llc -mtriple arm64-none-eabi -global-isel -o - %s | FileCheck %s --check-prefixes=CHECK,GISEL +; RUN: llc -mtriple arm64-none-eabi -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple arm64-none-eabi -global-isel -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI @out = internal global i32 0, align 4 ; Ensure that we transform select(C0, x, select(C1, x, y)) towards ; select(C0 | C1, x, y) so we can use CMP;CCMP for the implementation. define i32 @test0(i32 %v0, i32 %v1, i32 %v2) { -; SDISEL-LABEL: test0: -; SDISEL: // %bb.0: -; SDISEL-NEXT: cmp w0, #7 -; SDISEL-NEXT: ccmp w1, #0, #0, ne -; SDISEL-NEXT: csel w0, w1, w2, gt -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: test0: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmp w0, #7 +; CHECK-SD-NEXT: ccmp w1, #0, #0, ne +; CHECK-SD-NEXT: csel w0, w1, w2, gt +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: test0: -; GISEL: // %bb.0: -; GISEL-NEXT: cmp w0, #7 -; GISEL-NEXT: csel w8, w1, w2, eq -; GISEL-NEXT: cmp w1, #0 -; GISEL-NEXT: csel w0, w1, w8, gt -; GISEL-NEXT: ret +; CHECK-GI-LABEL: test0: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w0, #7 +; CHECK-GI-NEXT: csel w8, w1, w2, eq +; CHECK-GI-NEXT: cmp w1, #0 +; CHECK-GI-NEXT: csel w0, w1, w8, gt +; CHECK-GI-NEXT: ret %cmp1 = icmp eq i32 %v0, 7 %cmp2 = icmp sgt i32 %v1, 0 %sel0 = select i1 %cmp1, i32 %v1, i32 %v2 @@ -32,36 +32,36 @@ define i32 @test0(i32 %v0, i32 %v1, i32 %v2) { ; sequences. This case should be transformed to select(C0, select(C1, x, y), y) ; anyway to get CSE effects. define void @test1(i32 %bitset, i32 %val0, i32 %val1) { -; SDISEL-LABEL: test1: -; SDISEL: // %bb.0: -; SDISEL-NEXT: cmp w0, #7 -; SDISEL-NEXT: adrp x9, out -; SDISEL-NEXT: csel w8, w1, w2, eq -; SDISEL-NEXT: cmp w8, #13 -; SDISEL-NEXT: csel w8, w1, w2, lo -; SDISEL-NEXT: cmp w0, #42 -; SDISEL-NEXT: csel w10, w1, w8, eq -; SDISEL-NEXT: str w8, [x9, :lo12:out] -; SDISEL-NEXT: str w10, [x9, :lo12:out] -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: test1: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmp w0, #7 +; CHECK-SD-NEXT: adrp x9, out +; CHECK-SD-NEXT: csel w8, w1, w2, eq +; CHECK-SD-NEXT: cmp w8, #13 +; CHECK-SD-NEXT: csel w8, w1, w2, lo +; CHECK-SD-NEXT: cmp w0, #42 +; CHECK-SD-NEXT: csel w10, w1, w8, eq +; CHECK-SD-NEXT: str w8, [x9, :lo12:out] +; CHECK-SD-NEXT: str w10, [x9, :lo12:out] +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: test1: -; GISEL: // %bb.0: -; GISEL-NEXT: cmp w0, #7 -; GISEL-NEXT: csel w8, w1, w2, eq -; GISEL-NEXT: cmp w8, #13 -; GISEL-NEXT: cset w8, lo -; GISEL-NEXT: tst w8, #0x1 -; GISEL-NEXT: csel w9, w1, w2, ne -; GISEL-NEXT: cmp w0, #42 -; GISEL-NEXT: cset w10, eq -; GISEL-NEXT: orr w8, w10, w8 -; GISEL-NEXT: tst w8, #0x1 -; GISEL-NEXT: adrp x8, out -; GISEL-NEXT: csel w10, w1, w2, ne -; GISEL-NEXT: str w9, [x8, :lo12:out] -; GISEL-NEXT: str w10, [x8, :lo12:out] -; GISEL-NEXT: ret +; CHECK-GI-LABEL: test1: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w0, #7 +; CHECK-GI-NEXT: csel w8, w1, w2, eq +; CHECK-GI-NEXT: cmp w8, #13 +; CHECK-GI-NEXT: cset w8, lo +; CHECK-GI-NEXT: tst w8, #0x1 +; CHECK-GI-NEXT: csel w9, w1, w2, ne +; CHECK-GI-NEXT: cmp w0, #42 +; CHECK-GI-NEXT: cset w10, eq +; CHECK-GI-NEXT: orr w8, w10, w8 +; CHECK-GI-NEXT: tst w8, #0x1 +; CHECK-GI-NEXT: adrp x8, out +; CHECK-GI-NEXT: csel w10, w1, w2, ne +; CHECK-GI-NEXT: str w9, [x8, :lo12:out] +; CHECK-GI-NEXT: str w10, [x8, :lo12:out] +; CHECK-GI-NEXT: ret %cmp1 = icmp eq i32 %bitset, 7 %cond = select i1 %cmp1, i32 %val0, i32 %val1 %cmp5 = icmp ult i32 %cond, 13 diff --git a/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll b/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll index 4906e2e..c6b8e41 100644 --- a/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll +++ b/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll @@ -1431,7 +1431,6 @@ define <9 x half> @max_v9f16(<9 x half> %a, <9 x half> %b) { ; FULLFP16-NEXT: add x9, sp, #16 ; FULLFP16-NEXT: // kill: def $h3 killed $h3 def $q3 ; FULLFP16-NEXT: // kill: def $h4 killed $h4 def $q4 -; FULLFP16-NEXT: add x10, sp, #40 ; FULLFP16-NEXT: // kill: def $h5 killed $h5 def $q5 ; FULLFP16-NEXT: // kill: def $h6 killed $h6 def $q6 ; FULLFP16-NEXT: // kill: def $h7 killed $h7 def $q7 @@ -1440,30 +1439,30 @@ define <9 x half> @max_v9f16(<9 x half> %a, <9 x half> %b) { ; FULLFP16-NEXT: ld1 { v1.h }[1], [x9] ; FULLFP16-NEXT: add x9, sp, #24 ; FULLFP16-NEXT: mov v0.h[2], v2.h[0] +; FULLFP16-NEXT: ldr h2, [sp] ; FULLFP16-NEXT: ld1 { v1.h }[2], [x9] ; FULLFP16-NEXT: add x9, sp, #32 +; FULLFP16-NEXT: fminnm v2.8h, v2.8h, v2.8h ; FULLFP16-NEXT: mov v0.h[3], v3.h[0] ; FULLFP16-NEXT: ld1 { v1.h }[3], [x9] -; FULLFP16-NEXT: ldr h2, [x10] -; FULLFP16-NEXT: add x9, sp, #48 +; FULLFP16-NEXT: add x9, sp, #40 ; FULLFP16-NEXT: ldr h3, [sp, #72] -; FULLFP16-NEXT: ld1 { v2.h }[1], [x9] -; FULLFP16-NEXT: add x9, sp, #56 +; FULLFP16-NEXT: ld1 { v1.h }[4], [x9] +; FULLFP16-NEXT: add x9, sp, #48 ; FULLFP16-NEXT: fminnm v3.8h, v3.8h, v3.8h ; FULLFP16-NEXT: mov v0.h[4], v4.h[0] -; FULLFP16-NEXT: ld1 { v2.h }[2], [x9] -; FULLFP16-NEXT: add x9, sp, #64 +; FULLFP16-NEXT: ld1 { v1.h }[5], [x9] +; FULLFP16-NEXT: add x9, sp, #56 +; FULLFP16-NEXT: fmaxnm v2.8h, v2.8h, v3.8h ; FULLFP16-NEXT: mov v0.h[5], v5.h[0] -; FULLFP16-NEXT: ld1 { v2.h }[3], [x9] -; FULLFP16-NEXT: zip1 v1.2d, v1.2d, v2.2d -; FULLFP16-NEXT: ldr h2, [sp] +; FULLFP16-NEXT: ld1 { v1.h }[6], [x9] +; FULLFP16-NEXT: add x9, sp, #64 +; FULLFP16-NEXT: str h2, [x8, #16] ; FULLFP16-NEXT: mov v0.h[6], v6.h[0] -; FULLFP16-NEXT: fminnm v2.8h, v2.8h, v2.8h +; FULLFP16-NEXT: ld1 { v1.h }[7], [x9] ; FULLFP16-NEXT: fminnm v1.8h, v1.8h, v1.8h ; FULLFP16-NEXT: mov v0.h[7], v7.h[0] -; FULLFP16-NEXT: fmaxnm v2.8h, v2.8h, v3.8h ; FULLFP16-NEXT: fminnm v0.8h, v0.8h, v0.8h -; FULLFP16-NEXT: str h2, [x8, #16] ; FULLFP16-NEXT: fmaxnm v0.8h, v0.8h, v1.8h ; FULLFP16-NEXT: str q0, [x8] ; FULLFP16-NEXT: ret @@ -2013,7 +2012,6 @@ define <9 x half> @min_v9f16(<9 x half> %a, <9 x half> %b) { ; FULLFP16-NEXT: add x9, sp, #16 ; FULLFP16-NEXT: // kill: def $h3 killed $h3 def $q3 ; FULLFP16-NEXT: // kill: def $h4 killed $h4 def $q4 -; FULLFP16-NEXT: add x10, sp, #40 ; FULLFP16-NEXT: // kill: def $h5 killed $h5 def $q5 ; FULLFP16-NEXT: // kill: def $h6 killed $h6 def $q6 ; FULLFP16-NEXT: // kill: def $h7 killed $h7 def $q7 @@ -2022,30 +2020,30 @@ define <9 x half> @min_v9f16(<9 x half> %a, <9 x half> %b) { ; FULLFP16-NEXT: ld1 { v1.h }[1], [x9] ; FULLFP16-NEXT: add x9, sp, #24 ; FULLFP16-NEXT: mov v0.h[2], v2.h[0] +; FULLFP16-NEXT: ldr h2, [sp] ; FULLFP16-NEXT: ld1 { v1.h }[2], [x9] ; FULLFP16-NEXT: add x9, sp, #32 +; FULLFP16-NEXT: fminnm v2.8h, v2.8h, v2.8h ; FULLFP16-NEXT: mov v0.h[3], v3.h[0] ; FULLFP16-NEXT: ld1 { v1.h }[3], [x9] -; FULLFP16-NEXT: ldr h2, [x10] -; FULLFP16-NEXT: add x9, sp, #48 +; FULLFP16-NEXT: add x9, sp, #40 ; FULLFP16-NEXT: ldr h3, [sp, #72] -; FULLFP16-NEXT: ld1 { v2.h }[1], [x9] -; FULLFP16-NEXT: add x9, sp, #56 +; FULLFP16-NEXT: ld1 { v1.h }[4], [x9] +; FULLFP16-NEXT: add x9, sp, #48 ; FULLFP16-NEXT: fminnm v3.8h, v3.8h, v3.8h ; FULLFP16-NEXT: mov v0.h[4], v4.h[0] -; FULLFP16-NEXT: ld1 { v2.h }[2], [x9] -; FULLFP16-NEXT: add x9, sp, #64 +; FULLFP16-NEXT: ld1 { v1.h }[5], [x9] +; FULLFP16-NEXT: add x9, sp, #56 +; FULLFP16-NEXT: fminnm v2.8h, v2.8h, v3.8h ; FULLFP16-NEXT: mov v0.h[5], v5.h[0] -; FULLFP16-NEXT: ld1 { v2.h }[3], [x9] -; FULLFP16-NEXT: zip1 v1.2d, v1.2d, v2.2d -; FULLFP16-NEXT: ldr h2, [sp] +; FULLFP16-NEXT: ld1 { v1.h }[6], [x9] +; FULLFP16-NEXT: add x9, sp, #64 +; FULLFP16-NEXT: str h2, [x8, #16] ; FULLFP16-NEXT: mov v0.h[6], v6.h[0] -; FULLFP16-NEXT: fminnm v2.8h, v2.8h, v2.8h +; FULLFP16-NEXT: ld1 { v1.h }[7], [x9] ; FULLFP16-NEXT: fminnm v1.8h, v1.8h, v1.8h ; FULLFP16-NEXT: mov v0.h[7], v7.h[0] -; FULLFP16-NEXT: fminnm v2.8h, v2.8h, v3.8h ; FULLFP16-NEXT: fminnm v0.8h, v0.8h, v0.8h -; FULLFP16-NEXT: str h2, [x8, #16] ; FULLFP16-NEXT: fminnm v0.8h, v0.8h, v1.8h ; FULLFP16-NEXT: str q0, [x8] ; FULLFP16-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll index 1b98954..b056460 100644 --- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll +++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64 -global-isel=0 -mattr=+v8.2a,+fullfp16 | FileCheck %s --check-prefixes=CHECK,SDISEL -; RUN: llc < %s -mtriple=aarch64 -global-isel=1 -mattr=+v8.2a,+fullfp16 | FileCheck %s --check-prefixes=CHECK,GISEL +; RUN: llc < %s -mtriple=aarch64 -global-isel=0 -mattr=+v8.2a,+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc < %s -mtriple=aarch64 -global-isel=1 -mattr=+v8.2a,+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-GI declare i64 @llvm.aarch64.neon.fcvtpu.i64.f16(half) declare i32 @llvm.aarch64.neon.fcvtpu.i32.f16(half) @@ -27,18 +27,18 @@ declare half @llvm.aarch64.neon.frecpx.f16(half) declare half @llvm.aarch64.neon.frecpe.f16(half) define dso_local i16 @t2(half %a) { -; SDISEL-LABEL: t2: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fcmp h0, #0.0 -; SDISEL-NEXT: csetm w0, eq -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: t2: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fcmp h0, #0.0 +; CHECK-SD-NEXT: csetm w0, eq +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: t2: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: fcmp h0, #0.0 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: sbfx w0, w8, #0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: t2: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: fcmp h0, #0.0 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: sbfx w0, w8, #0, #1 +; CHECK-GI-NEXT: ret entry: %0 = fcmp oeq half %a, 0xH0000 %vceqz = sext i1 %0 to i16 @@ -46,18 +46,18 @@ entry: } define dso_local i16 @t3(half %a) { -; SDISEL-LABEL: t3: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fcmp h0, #0.0 -; SDISEL-NEXT: csetm w0, ge -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: t3: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fcmp h0, #0.0 +; CHECK-SD-NEXT: csetm w0, ge +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: t3: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: fcmp h0, #0.0 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: sbfx w0, w8, #0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: t3: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: fcmp h0, #0.0 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: sbfx w0, w8, #0, #1 +; CHECK-GI-NEXT: ret entry: %0 = fcmp oge half %a, 0xH0000 %vcgez = sext i1 %0 to i16 @@ -65,18 +65,18 @@ entry: } define dso_local i16 @t4(half %a) { -; SDISEL-LABEL: t4: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fcmp h0, #0.0 -; SDISEL-NEXT: csetm w0, gt -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: t4: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fcmp h0, #0.0 +; CHECK-SD-NEXT: csetm w0, gt +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: t4: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: fcmp h0, #0.0 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: sbfx w0, w8, #0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: t4: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: fcmp h0, #0.0 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: sbfx w0, w8, #0, #1 +; CHECK-GI-NEXT: ret entry: %0 = fcmp ogt half %a, 0xH0000 %vcgtz = sext i1 %0 to i16 @@ -84,18 +84,18 @@ entry: } define dso_local i16 @t5(half %a) { -; SDISEL-LABEL: t5: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fcmp h0, #0.0 -; SDISEL-NEXT: csetm w0, ls -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: t5: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fcmp h0, #0.0 +; CHECK-SD-NEXT: csetm w0, ls +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: t5: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: fcmp h0, #0.0 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: sbfx w0, w8, #0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: t5: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: fcmp h0, #0.0 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: sbfx w0, w8, #0, #1 +; CHECK-GI-NEXT: ret entry: %0 = fcmp ole half %a, 0xH0000 %vclez = sext i1 %0 to i16 @@ -103,18 +103,18 @@ entry: } define dso_local i16 @t6(half %a) { -; SDISEL-LABEL: t6: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fcmp h0, #0.0 -; SDISEL-NEXT: csetm w0, mi -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: t6: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fcmp h0, #0.0 +; CHECK-SD-NEXT: csetm w0, mi +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: t6: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: fcmp h0, #0.0 -; GISEL-NEXT: cset w8, mi -; GISEL-NEXT: sbfx w0, w8, #0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: t6: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: fcmp h0, #0.0 +; CHECK-GI-NEXT: cset w8, mi +; CHECK-GI-NEXT: sbfx w0, w8, #0, #1 +; CHECK-GI-NEXT: ret entry: %0 = fcmp olt half %a, 0xH0000 %vcltz = sext i1 %0 to i16 @@ -172,15 +172,15 @@ entry: } define dso_local i16 @t16(half %a) { -; SDISEL-LABEL: t16: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fcvtzs w0, h0 -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: t16: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fcvtzs w0, h0 +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: t16: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: fcvtzu w0, h0 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: t16: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: fcvtzu w0, h0 +; CHECK-GI-NEXT: ret entry: %0 = fptoui half %a to i16 ret i16 %0 diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_2op.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_2op.ll index 5b08ef2..da70599 100644 --- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_2op.ll +++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_2op.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64 -global-isel=0 -mattr=+v8.2a,+fullfp16 | FileCheck %s --check-prefixes=CHECK,SDISEL -; RUN: llc < %s -mtriple=aarch64 -global-isel=1 -mattr=+v8.2a,+fullfp16 | FileCheck %s --check-prefixes=CHECK,GISEL +; RUN: llc < %s -mtriple=aarch64 -global-isel=0 -mattr=+v8.2a,+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc < %s -mtriple=aarch64 -global-isel=1 -mattr=+v8.2a,+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-GI declare half @llvm.aarch64.sisd.fabd.f16(half, half) @@ -35,18 +35,18 @@ entry: } define dso_local i16 @t_vceqh_f16(half %a, half %b) { -; SDISEL-LABEL: t_vceqh_f16: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fcmp h0, h1 -; SDISEL-NEXT: csetm w0, eq -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: t_vceqh_f16: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fcmp h0, h1 +; CHECK-SD-NEXT: csetm w0, eq +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: t_vceqh_f16: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: fcmp h0, h1 -; GISEL-NEXT: cset w8, eq -; GISEL-NEXT: sbfx w0, w8, #0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: t_vceqh_f16: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: fcmp h0, h1 +; CHECK-GI-NEXT: cset w8, eq +; CHECK-GI-NEXT: sbfx w0, w8, #0, #1 +; CHECK-GI-NEXT: ret entry: %0 = fcmp oeq half %a, %b %vcmpd = sext i1 %0 to i16 @@ -54,18 +54,18 @@ entry: } define dso_local i16 @t_vcgeh_f16(half %a, half %b) { -; SDISEL-LABEL: t_vcgeh_f16: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fcmp h0, h1 -; SDISEL-NEXT: csetm w0, ge -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: t_vcgeh_f16: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fcmp h0, h1 +; CHECK-SD-NEXT: csetm w0, ge +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: t_vcgeh_f16: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: fcmp h0, h1 -; GISEL-NEXT: cset w8, ge -; GISEL-NEXT: sbfx w0, w8, #0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: t_vcgeh_f16: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: fcmp h0, h1 +; CHECK-GI-NEXT: cset w8, ge +; CHECK-GI-NEXT: sbfx w0, w8, #0, #1 +; CHECK-GI-NEXT: ret entry: %0 = fcmp oge half %a, %b %vcmpd = sext i1 %0 to i16 @@ -73,18 +73,18 @@ entry: } define dso_local i16 @t_vcgth_f16(half %a, half %b) { -; SDISEL-LABEL: t_vcgth_f16: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fcmp h0, h1 -; SDISEL-NEXT: csetm w0, gt -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: t_vcgth_f16: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fcmp h0, h1 +; CHECK-SD-NEXT: csetm w0, gt +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: t_vcgth_f16: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: fcmp h0, h1 -; GISEL-NEXT: cset w8, gt -; GISEL-NEXT: sbfx w0, w8, #0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: t_vcgth_f16: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: fcmp h0, h1 +; CHECK-GI-NEXT: cset w8, gt +; CHECK-GI-NEXT: sbfx w0, w8, #0, #1 +; CHECK-GI-NEXT: ret entry: %0 = fcmp ogt half %a, %b %vcmpd = sext i1 %0 to i16 @@ -92,18 +92,18 @@ entry: } define dso_local i16 @t_vcleh_f16(half %a, half %b) { -; SDISEL-LABEL: t_vcleh_f16: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fcmp h0, h1 -; SDISEL-NEXT: csetm w0, ls -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: t_vcleh_f16: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fcmp h0, h1 +; CHECK-SD-NEXT: csetm w0, ls +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: t_vcleh_f16: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: fcmp h0, h1 -; GISEL-NEXT: cset w8, ls -; GISEL-NEXT: sbfx w0, w8, #0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: t_vcleh_f16: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: fcmp h0, h1 +; CHECK-GI-NEXT: cset w8, ls +; CHECK-GI-NEXT: sbfx w0, w8, #0, #1 +; CHECK-GI-NEXT: ret entry: %0 = fcmp ole half %a, %b %vcmpd = sext i1 %0 to i16 @@ -111,18 +111,18 @@ entry: } define dso_local i16 @t_vclth_f16(half %a, half %b) { -; SDISEL-LABEL: t_vclth_f16: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fcmp h0, h1 -; SDISEL-NEXT: csetm w0, mi -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: t_vclth_f16: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fcmp h0, h1 +; CHECK-SD-NEXT: csetm w0, mi +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: t_vclth_f16: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: fcmp h0, h1 -; GISEL-NEXT: cset w8, mi -; GISEL-NEXT: sbfx w0, w8, #0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: t_vclth_f16: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: fcmp h0, h1 +; CHECK-GI-NEXT: cset w8, mi +; CHECK-GI-NEXT: sbfx w0, w8, #0, #1 +; CHECK-GI-NEXT: ret entry: %0 = fcmp olt half %a, %b %vcmpd = sext i1 %0 to i16 @@ -187,18 +187,18 @@ declare half @llvm.aarch64.neon.vcvtfxu2fp.f16.i32(i32, i32) #1 declare i32 @llvm.aarch64.neon.vcvtfp2fxu.i32.f16(half, i32) #1 define dso_local half @test_vcvth_n_f16_s16_1(i16 %a) { -; SDISEL-LABEL: test_vcvth_n_f16_s16_1: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fmov s0, w0 -; SDISEL-NEXT: scvtf h0, h0, #1 -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: test_vcvth_n_f16_s16_1: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fmov s0, w0 +; CHECK-SD-NEXT: scvtf h0, h0, #1 +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: test_vcvth_n_f16_s16_1: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: sxth w8, w0 -; GISEL-NEXT: fmov s0, w8 -; GISEL-NEXT: scvtf h0, h0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: test_vcvth_n_f16_s16_1: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sxth w8, w0 +; CHECK-GI-NEXT: fmov s0, w8 +; CHECK-GI-NEXT: scvtf h0, h0, #1 +; CHECK-GI-NEXT: ret entry: %sext = sext i16 %a to i32 %fcvth_n = tail call half @llvm.aarch64.neon.vcvtfxs2fp.f16.i32(i32 %sext, i32 1) @@ -206,18 +206,18 @@ entry: } define dso_local half @test_vcvth_n_f16_s16_16(i16 %a) { -; SDISEL-LABEL: test_vcvth_n_f16_s16_16: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fmov s0, w0 -; SDISEL-NEXT: scvtf h0, h0, #16 -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: test_vcvth_n_f16_s16_16: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fmov s0, w0 +; CHECK-SD-NEXT: scvtf h0, h0, #16 +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: test_vcvth_n_f16_s16_16: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: sxth w8, w0 -; GISEL-NEXT: fmov s0, w8 -; GISEL-NEXT: scvtf h0, h0, #16 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: test_vcvth_n_f16_s16_16: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sxth w8, w0 +; CHECK-GI-NEXT: fmov s0, w8 +; CHECK-GI-NEXT: scvtf h0, h0, #16 +; CHECK-GI-NEXT: ret entry: %sext = sext i16 %a to i32 %fcvth_n = tail call half @llvm.aarch64.neon.vcvtfxs2fp.f16.i32(i32 %sext, i32 16) @@ -315,18 +315,18 @@ entry: } define dso_local half @test_vcvth_n_f16_u16_1(i16 %a) { -; SDISEL-LABEL: test_vcvth_n_f16_u16_1: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fmov s0, w0 -; SDISEL-NEXT: ucvtf h0, h0, #1 -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: test_vcvth_n_f16_u16_1: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fmov s0, w0 +; CHECK-SD-NEXT: ucvtf h0, h0, #1 +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: test_vcvth_n_f16_u16_1: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: and w8, w0, #0xffff -; GISEL-NEXT: fmov s0, w8 -; GISEL-NEXT: ucvtf h0, h0, #1 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: test_vcvth_n_f16_u16_1: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: and w8, w0, #0xffff +; CHECK-GI-NEXT: fmov s0, w8 +; CHECK-GI-NEXT: ucvtf h0, h0, #1 +; CHECK-GI-NEXT: ret entry: %0 = zext i16 %a to i32 %fcvth_n = tail call half @llvm.aarch64.neon.vcvtfxu2fp.f16.i32(i32 %0, i32 1) @@ -334,18 +334,18 @@ entry: } define dso_local half @test_vcvth_n_f16_u16_16(i16 %a) { -; SDISEL-LABEL: test_vcvth_n_f16_u16_16: -; SDISEL: // %bb.0: // %entry -; SDISEL-NEXT: fmov s0, w0 -; SDISEL-NEXT: ucvtf h0, h0, #16 -; SDISEL-NEXT: ret +; CHECK-SD-LABEL: test_vcvth_n_f16_u16_16: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: fmov s0, w0 +; CHECK-SD-NEXT: ucvtf h0, h0, #16 +; CHECK-SD-NEXT: ret ; -; GISEL-LABEL: test_vcvth_n_f16_u16_16: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: and w8, w0, #0xffff -; GISEL-NEXT: fmov s0, w8 -; GISEL-NEXT: ucvtf h0, h0, #16 -; GISEL-NEXT: ret +; CHECK-GI-LABEL: test_vcvth_n_f16_u16_16: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: and w8, w0, #0xffff +; CHECK-GI-NEXT: fmov s0, w8 +; CHECK-GI-NEXT: ucvtf h0, h0, #16 +; CHECK-GI-NEXT: ret entry: %0 = zext i16 %a to i32 %fcvth_n = tail call half @llvm.aarch64.neon.vcvtfxu2fp.f16.i32(i32 %0, i32 16) diff --git a/llvm/test/CodeGen/AArch64/fsh.ll b/llvm/test/CodeGen/AArch64/fsh.ll index ae2ef26..4c28c90 100644 --- a/llvm/test/CodeGen/AArch64/fsh.ll +++ b/llvm/test/CodeGen/AArch64/fsh.ll @@ -2509,88 +2509,87 @@ define <7 x i32> @fshl_v7i32(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) { ; ; CHECK-GI-LABEL: fshl_v7i32: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ldr s17, [sp, #48] -; CHECK-GI-NEXT: add x8, sp, #56 -; CHECK-GI-NEXT: add x9, sp, #64 +; CHECK-GI-NEXT: ldr s3, [sp, #48] +; CHECK-GI-NEXT: ldr s20, [sp, #56] +; CHECK-GI-NEXT: add x9, sp, #56 ; CHECK-GI-NEXT: ldr s4, [sp, #48] -; CHECK-GI-NEXT: ldr s21, [sp, #56] -; CHECK-GI-NEXT: mov w10, #-1 // =0xffffffff -; CHECK-GI-NEXT: ld1 { v17.s }[1], [x8] -; CHECK-GI-NEXT: ldr s20, [x9] -; CHECK-GI-NEXT: add x8, sp, #72 -; CHECK-GI-NEXT: mov v4.s[1], v21.s[0] +; CHECK-GI-NEXT: ldr s7, [sp, #80] +; CHECK-GI-NEXT: mov w12, #-1 // =0xffffffff +; CHECK-GI-NEXT: ldr s21, [sp, #88] +; CHECK-GI-NEXT: mov v3.s[1], v20.s[0] +; CHECK-GI-NEXT: fmov s20, w12 +; CHECK-GI-NEXT: ld1 { v4.s }[1], [x9] +; CHECK-GI-NEXT: ldr s17, [sp] +; CHECK-GI-NEXT: add x13, sp, #64 +; CHECK-GI-NEXT: mov v7.s[1], v21.s[0] ; CHECK-GI-NEXT: fmov s21, w7 -; CHECK-GI-NEXT: ldr s6, [sp] -; CHECK-GI-NEXT: ld1 { v20.s }[1], [x8] ; CHECK-GI-NEXT: ldr s19, [sp, #64] -; CHECK-GI-NEXT: ldr s7, [sp, #80] -; CHECK-GI-NEXT: ldr s22, [sp, #88] -; CHECK-GI-NEXT: mov w9, #31 // =0x1f -; CHECK-GI-NEXT: mov w11, #1 // =0x1 -; CHECK-GI-NEXT: mov v21.s[1], v6.s[0] -; CHECK-GI-NEXT: fmov s6, w9 +; CHECK-GI-NEXT: mov w11, #31 // =0x1f +; CHECK-GI-NEXT: mov v20.s[1], w12 ; CHECK-GI-NEXT: ldr s18, [sp, #96] -; CHECK-GI-NEXT: zip1 v17.2d, v17.2d, v20.2d -; CHECK-GI-NEXT: fmov s20, w10 -; CHECK-GI-NEXT: mov v7.s[1], v22.s[0] -; CHECK-GI-NEXT: mov v4.s[2], v19.s[0] -; CHECK-GI-NEXT: fmov s19, w11 +; CHECK-GI-NEXT: ld1 { v4.s }[2], [x13] +; CHECK-GI-NEXT: mov w13, #1 // =0x1 +; CHECK-GI-NEXT: mov v3.s[2], v19.s[0] +; CHECK-GI-NEXT: mov v21.s[1], v17.s[0] +; CHECK-GI-NEXT: fmov s17, w11 +; CHECK-GI-NEXT: fmov s19, w13 ; CHECK-GI-NEXT: fmov s23, w0 -; CHECK-GI-NEXT: mov v6.s[1], w9 -; CHECK-GI-NEXT: fmov s24, w9 -; CHECK-GI-NEXT: ldr s2, [sp, #8] -; CHECK-GI-NEXT: mov v20.s[1], w10 +; CHECK-GI-NEXT: fmov s24, w11 +; CHECK-GI-NEXT: ldr s6, [sp, #8] ; CHECK-GI-NEXT: ldr s0, [sp, #24] ; CHECK-GI-NEXT: ldr s5, [sp, #32] -; CHECK-GI-NEXT: mov v19.s[1], w11 ; CHECK-GI-NEXT: mov v7.s[2], v18.s[0] +; CHECK-GI-NEXT: mov v17.s[1], w11 +; CHECK-GI-NEXT: mov v19.s[1], w13 +; CHECK-GI-NEXT: mov v20.s[2], w12 ; CHECK-GI-NEXT: ldr s16, [sp, #72] ; CHECK-GI-NEXT: mov v23.s[1], w1 ; CHECK-GI-NEXT: ldr s18, [sp, #80] -; CHECK-GI-NEXT: mov v21.s[2], v2.s[0] -; CHECK-GI-NEXT: mov v24.s[1], w9 +; CHECK-GI-NEXT: mov v21.s[2], v6.s[0] +; CHECK-GI-NEXT: mov v24.s[1], w11 ; CHECK-GI-NEXT: mov v0.s[1], v5.s[0] -; CHECK-GI-NEXT: fmov s5, w4 -; CHECK-GI-NEXT: mov v20.s[2], w10 -; CHECK-GI-NEXT: add x8, sp, #88 +; CHECK-GI-NEXT: fmov s6, w4 +; CHECK-GI-NEXT: add x10, sp, #88 ; CHECK-GI-NEXT: movi v22.4s, #31 -; CHECK-GI-NEXT: mov v4.s[3], v16.s[0] -; CHECK-GI-NEXT: mov v6.s[2], w9 -; CHECK-GI-NEXT: mov v19.s[2], w11 -; CHECK-GI-NEXT: ldr s1, [sp, #16] -; CHECK-GI-NEXT: ldr s3, [sp, #40] -; CHECK-GI-NEXT: ld1 { v18.s }[1], [x8] +; CHECK-GI-NEXT: mov v3.s[3], v16.s[0] +; CHECK-GI-NEXT: mov v17.s[2], w11 +; CHECK-GI-NEXT: mov v19.s[2], w13 +; CHECK-GI-NEXT: ldr s2, [sp, #16] +; CHECK-GI-NEXT: ldr s1, [sp, #40] +; CHECK-GI-NEXT: ld1 { v18.s }[1], [x10] +; CHECK-GI-NEXT: eor v5.16b, v7.16b, v20.16b ; CHECK-GI-NEXT: mov v23.s[2], w2 -; CHECK-GI-NEXT: mov v5.s[1], w5 -; CHECK-GI-NEXT: add x8, sp, #96 -; CHECK-GI-NEXT: eor v2.16b, v7.16b, v20.16b -; CHECK-GI-NEXT: mov v21.s[3], v1.s[0] -; CHECK-GI-NEXT: mov v24.s[2], w9 -; CHECK-GI-NEXT: mov v0.s[2], v3.s[0] -; CHECK-GI-NEXT: bic v1.16b, v22.16b, v4.16b -; CHECK-GI-NEXT: ld1 { v18.s }[2], [x8] +; CHECK-GI-NEXT: mov v6.s[1], w5 +; CHECK-GI-NEXT: add x8, sp, #72 +; CHECK-GI-NEXT: add x9, sp, #96 +; CHECK-GI-NEXT: mov v21.s[3], v2.s[0] +; CHECK-GI-NEXT: mov v24.s[2], w11 +; CHECK-GI-NEXT: mov v0.s[2], v1.s[0] +; CHECK-GI-NEXT: ld1 { v4.s }[3], [x8] +; CHECK-GI-NEXT: bic v2.16b, v22.16b, v3.16b +; CHECK-GI-NEXT: ld1 { v18.s }[2], [x9] +; CHECK-GI-NEXT: and v1.16b, v5.16b, v17.16b ; CHECK-GI-NEXT: neg v3.4s, v19.4s -; CHECK-GI-NEXT: and v4.16b, v17.16b, v22.16b -; CHECK-GI-NEXT: and v2.16b, v2.16b, v6.16b ; CHECK-GI-NEXT: mov v23.s[3], w3 -; CHECK-GI-NEXT: mov v5.s[2], w6 -; CHECK-GI-NEXT: ushr v6.4s, v21.4s, #1 -; CHECK-GI-NEXT: neg v1.4s, v1.4s +; CHECK-GI-NEXT: mov v6.s[2], w6 +; CHECK-GI-NEXT: and v4.16b, v4.16b, v22.16b +; CHECK-GI-NEXT: ushr v5.4s, v21.4s, #1 +; CHECK-GI-NEXT: neg v2.4s, v2.4s ; CHECK-GI-NEXT: and v7.16b, v18.16b, v24.16b +; CHECK-GI-NEXT: neg v1.4s, v1.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v3.4s -; CHECK-GI-NEXT: neg v2.4s, v2.4s ; CHECK-GI-NEXT: ushl v3.4s, v23.4s, v4.4s -; CHECK-GI-NEXT: ushl v1.4s, v6.4s, v1.4s -; CHECK-GI-NEXT: ushl v4.4s, v5.4s, v7.4s -; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v2.4s -; CHECK-GI-NEXT: orr v1.16b, v3.16b, v1.16b +; CHECK-GI-NEXT: ushl v2.4s, v5.4s, v2.4s +; CHECK-GI-NEXT: ushl v4.4s, v6.4s, v7.4s +; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: orr v1.16b, v3.16b, v2.16b ; CHECK-GI-NEXT: orr v0.16b, v4.16b, v0.16b ; CHECK-GI-NEXT: mov s2, v1.s[1] ; CHECK-GI-NEXT: mov s3, v1.s[2] ; CHECK-GI-NEXT: mov s4, v1.s[3] -; CHECK-GI-NEXT: fmov w0, s1 ; CHECK-GI-NEXT: mov s5, v0.s[1] ; CHECK-GI-NEXT: mov s6, v0.s[2] +; CHECK-GI-NEXT: fmov w0, s1 ; CHECK-GI-NEXT: fmov w4, s0 ; CHECK-GI-NEXT: fmov w1, s2 ; CHECK-GI-NEXT: fmov w2, s3 diff --git a/llvm/test/CodeGen/AArch64/llvm.frexp.ll b/llvm/test/CodeGen/AArch64/llvm.frexp.ll index 4e1876d..2213aa1 100644 --- a/llvm/test/CodeGen/AArch64/llvm.frexp.ll +++ b/llvm/test/CodeGen/AArch64/llvm.frexp.ll @@ -700,14 +700,13 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) nounwi ; CHECK-NEXT: ldr s1, [sp, #44] ; CHECK-NEXT: ldr q2, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0 -; CHECK-NEXT: mov v2.s[3], v0.s[0] ; CHECK-NEXT: ld1 { v1.s }[1], [x19] -; CHECK-NEXT: ldr s0, [x20] -; CHECK-NEXT: ld1 { v0.s }[1], [x21] +; CHECK-NEXT: mov v2.s[3], v0.s[0] +; CHECK-NEXT: ld1 { v1.s }[2], [x20] ; CHECK-NEXT: ldp x20, x19, [sp, #64] // 16-byte Folded Reload -; CHECK-NEXT: ldp x30, x21, [sp, #48] // 16-byte Folded Reload -; CHECK-NEXT: zip1 v1.2d, v1.2d, v0.2d ; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v1.s }[3], [x21] +; CHECK-NEXT: ldp x30, x21, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #80 ; CHECK-NEXT: ret ; @@ -873,11 +872,10 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) nounwind { ; CHECK-NEXT: bl frexpf ; CHECK-NEXT: ldr s0, [sp, #28] ; CHECK-NEXT: ld1 { v0.s }[1], [x19] -; CHECK-NEXT: ldr s1, [x20] -; CHECK-NEXT: ld1 { v1.s }[1], [x21] +; CHECK-NEXT: ld1 { v0.s }[2], [x20] ; CHECK-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; CHECK-NEXT: ld1 { v0.s }[3], [x21] ; CHECK-NEXT: ldp x30, x21, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: zip1 v0.2d, v0.2d, v1.2d ; CHECK-NEXT: add sp, sp, #64 ; CHECK-NEXT: ret ; diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll index 9443004..4f0c408 100644 --- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll +++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll @@ -6810,200 +6810,195 @@ define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> % ; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-SD-NEXT: .cfi_def_cfa_offset 16 ; CHECK-SD-NEXT: .cfi_offset w29, -16 -; CHECK-SD-NEXT: ldr b0, [sp, #208] +; CHECK-SD-NEXT: ldr b5, [sp, #208] ; CHECK-SD-NEXT: add x8, sp, #216 -; CHECK-SD-NEXT: add x9, sp, #272 -; CHECK-SD-NEXT: ldr b2, [sp, #80] +; CHECK-SD-NEXT: fmov s0, w0 ; CHECK-SD-NEXT: ldr b4, [sp, #976] -; CHECK-SD-NEXT: ldr b6, [sp, #720] -; CHECK-SD-NEXT: ld1 { v0.b }[1], [x8] +; CHECK-SD-NEXT: add x9, sp, #984 +; CHECK-SD-NEXT: add x12, sp, #328 +; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #224 -; CHECK-SD-NEXT: fmov s16, w0 -; CHECK-SD-NEXT: ldr b17, [sp, #848] -; CHECK-SD-NEXT: add x10, sp, #24 -; CHECK-SD-NEXT: movi v19.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v0.b }[2], [x8] +; CHECK-SD-NEXT: movi v1.16b, #1 +; CHECK-SD-NEXT: mov v0.b[1], w1 +; CHECK-SD-NEXT: ld1 { v4.b }[1], [x9] +; CHECK-SD-NEXT: movi v3.2d, #0000000000000000 +; CHECK-SD-NEXT: add x11, sp, #992 +; CHECK-SD-NEXT: ldr b6, [sp, #720] +; CHECK-SD-NEXT: ldr b7, [sp, #80] +; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8] ; CHECK-SD-NEXT: add x8, sp, #232 -; CHECK-SD-NEXT: mov v16.b[1], w1 -; CHECK-SD-NEXT: ld1 { v0.b }[3], [x8] +; CHECK-SD-NEXT: add x13, sp, #88 +; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11] +; CHECK-SD-NEXT: ld1 { v7.b }[1], [x13] +; CHECK-SD-NEXT: add x13, sp, #856 +; CHECK-SD-NEXT: mov v0.b[2], w2 +; CHECK-SD-NEXT: add x14, sp, #1008 +; CHECK-SD-NEXT: add x15, sp, #872 +; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8] ; CHECK-SD-NEXT: add x8, sp, #240 -; CHECK-SD-NEXT: mov v16.b[2], w2 -; CHECK-SD-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-SD-NEXT: add x16, sp, #888 +; CHECK-SD-NEXT: add x10, sp, #16 +; CHECK-SD-NEXT: add x9, sp, #24 +; CHECK-SD-NEXT: add x11, sp, #40 +; CHECK-SD-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8] ; CHECK-SD-NEXT: add x8, sp, #248 -; CHECK-SD-NEXT: mov v16.b[3], w3 -; CHECK-SD-NEXT: ld1 { v0.b }[5], [x8] +; CHECK-SD-NEXT: mov v0.b[3], w3 +; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8] ; CHECK-SD-NEXT: add x8, sp, #256 -; CHECK-SD-NEXT: ld1 { v0.b }[6], [x8] +; CHECK-SD-NEXT: mov v0.b[4], w4 +; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8] ; CHECK-SD-NEXT: add x8, sp, #264 -; CHECK-SD-NEXT: mov v16.b[4], w4 -; CHECK-SD-NEXT: ld1 { v0.b }[7], [x8] -; CHECK-SD-NEXT: ldr b1, [x9] +; CHECK-SD-NEXT: mov v0.b[5], w5 +; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8] +; CHECK-SD-NEXT: add x8, sp, #272 +; CHECK-SD-NEXT: ld1 { v5.b }[8], [x8] ; CHECK-SD-NEXT: add x8, sp, #280 -; CHECK-SD-NEXT: add x9, sp, #88 -; CHECK-SD-NEXT: mov v16.b[5], w5 -; CHECK-SD-NEXT: ld1 { v1.b }[1], [x8] +; CHECK-SD-NEXT: mov v0.b[6], w6 +; CHECK-SD-NEXT: ld1 { v5.b }[9], [x8] ; CHECK-SD-NEXT: add x8, sp, #288 -; CHECK-SD-NEXT: ld1 { v1.b }[2], [x8] +; CHECK-SD-NEXT: mov v0.b[7], w7 +; CHECK-SD-NEXT: ld1 { v5.b }[10], [x8] ; CHECK-SD-NEXT: add x8, sp, #296 -; CHECK-SD-NEXT: mov v16.b[6], w6 -; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8] +; CHECK-SD-NEXT: ld1 { v0.b }[8], [x10] +; CHECK-SD-NEXT: add x10, sp, #128 +; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8] ; CHECK-SD-NEXT: add x8, sp, #304 -; CHECK-SD-NEXT: mov v16.b[7], w7 -; CHECK-SD-NEXT: ld1 { v1.b }[4], [x8] +; CHECK-SD-NEXT: ld1 { v0.b }[9], [x9] +; CHECK-SD-NEXT: add x9, sp, #136 +; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8] ; CHECK-SD-NEXT: add x8, sp, #312 -; CHECK-SD-NEXT: ld1 { v1.b }[5], [x8] +; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8] ; CHECK-SD-NEXT: add x8, sp, #320 -; CHECK-SD-NEXT: ld1 { v1.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #328 -; CHECK-SD-NEXT: ld1 { v1.b }[7], [x8] -; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9] -; CHECK-SD-NEXT: add x8, sp, #96 -; CHECK-SD-NEXT: add x9, sp, #144 -; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #104 -; CHECK-SD-NEXT: zip1 v0.2d, v0.2d, v1.2d -; CHECK-SD-NEXT: movi v1.16b, #1 -; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #112 -; CHECK-SD-NEXT: ld1 { v2.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #120 -; CHECK-SD-NEXT: ld1 { v2.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #128 -; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #136 -; CHECK-SD-NEXT: ld1 { v2.b }[7], [x8] -; CHECK-SD-NEXT: ldr b3, [x9] +; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8] +; CHECK-SD-NEXT: add x8, sp, #32 +; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8] +; CHECK-SD-NEXT: add x8, sp, #144 +; CHECK-SD-NEXT: ld1 { v5.b }[15], [x12] +; CHECK-SD-NEXT: add x12, sp, #728 +; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12] +; CHECK-SD-NEXT: add x12, sp, #1000 +; CHECK-SD-NEXT: ld1 { v0.b }[11], [x11] +; CHECK-SD-NEXT: ld1 { v4.b }[3], [x12] +; CHECK-SD-NEXT: add x12, sp, #736 +; CHECK-SD-NEXT: add x11, sp, #920 +; CHECK-SD-NEXT: sdot v3.4s, v5.16b, v1.16b +; CHECK-SD-NEXT: ldr b5, [sp, #848] +; CHECK-SD-NEXT: ld1 { v6.b }[2], [x12] +; CHECK-SD-NEXT: add x12, sp, #48 +; CHECK-SD-NEXT: ld1 { v5.b }[1], [x13] +; CHECK-SD-NEXT: add x13, sp, #744 +; CHECK-SD-NEXT: ld1 { v4.b }[4], [x14] +; CHECK-SD-NEXT: add x14, sp, #96 +; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12] +; CHECK-SD-NEXT: ld1 { v6.b }[3], [x13] +; CHECK-SD-NEXT: add x13, sp, #864 +; CHECK-SD-NEXT: ld1 { v7.b }[2], [x14] +; CHECK-SD-NEXT: add x14, sp, #1016 +; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13] +; CHECK-SD-NEXT: add x13, sp, #752 +; CHECK-SD-NEXT: ld1 { v4.b }[5], [x14] +; CHECK-SD-NEXT: add x14, sp, #104 +; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13] +; CHECK-SD-NEXT: add x13, sp, #1024 +; CHECK-SD-NEXT: ld1 { v7.b }[3], [x14] +; CHECK-SD-NEXT: ld1 { v5.b }[3], [x15] +; CHECK-SD-NEXT: add x15, sp, #760 +; CHECK-SD-NEXT: add x14, sp, #112 +; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13] +; CHECK-SD-NEXT: add x13, sp, #880 +; CHECK-SD-NEXT: ld1 { v6.b }[5], [x15] +; CHECK-SD-NEXT: add x15, sp, #1032 +; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14] +; CHECK-SD-NEXT: ld1 { v5.b }[4], [x13] +; CHECK-SD-NEXT: add x14, sp, #768 +; CHECK-SD-NEXT: add x13, sp, #120 +; CHECK-SD-NEXT: ld1 { v4.b }[7], [x15] +; CHECK-SD-NEXT: add x15, sp, #1040 +; CHECK-SD-NEXT: ld1 { v6.b }[6], [x14] +; CHECK-SD-NEXT: ld1 { v7.b }[5], [x13] +; CHECK-SD-NEXT: add x13, sp, #776 +; CHECK-SD-NEXT: ld1 { v5.b }[5], [x16] +; CHECK-SD-NEXT: add x14, sp, #1048 +; CHECK-SD-NEXT: ld1 { v4.b }[8], [x15] +; CHECK-SD-NEXT: add x15, sp, #896 +; CHECK-SD-NEXT: ld1 { v6.b }[7], [x13] +; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10] +; CHECK-SD-NEXT: add x10, sp, #784 +; CHECK-SD-NEXT: ld1 { v5.b }[6], [x15] +; CHECK-SD-NEXT: add x13, sp, #1056 +; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14] +; CHECK-SD-NEXT: add x14, sp, #904 +; CHECK-SD-NEXT: ld1 { v6.b }[8], [x10] +; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9] +; CHECK-SD-NEXT: add x9, sp, #792 +; CHECK-SD-NEXT: ld1 { v5.b }[7], [x14] +; CHECK-SD-NEXT: add x10, sp, #1064 +; CHECK-SD-NEXT: ld1 { v4.b }[10], [x13] +; CHECK-SD-NEXT: add x13, sp, #912 +; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9] +; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8] +; CHECK-SD-NEXT: add x9, sp, #800 +; CHECK-SD-NEXT: ld1 { v5.b }[8], [x13] ; CHECK-SD-NEXT: add x8, sp, #152 -; CHECK-SD-NEXT: add x9, sp, #984 -; CHECK-SD-NEXT: ld1 { v3.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #160 -; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #168 -; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #176 -; CHECK-SD-NEXT: ld1 { v3.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #184 -; CHECK-SD-NEXT: ld1 { v3.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #192 -; CHECK-SD-NEXT: ld1 { v3.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #200 -; CHECK-SD-NEXT: ld1 { v3.b }[7], [x8] -; CHECK-SD-NEXT: ld1 { v4.b }[1], [x9] -; CHECK-SD-NEXT: add x8, sp, #992 -; CHECK-SD-NEXT: add x9, sp, #1040 -; CHECK-SD-NEXT: ld1 { v4.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #1000 -; CHECK-SD-NEXT: zip1 v2.2d, v2.2d, v3.2d -; CHECK-SD-NEXT: ld1 { v4.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #1008 -; CHECK-SD-NEXT: ld1 { v4.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #1016 -; CHECK-SD-NEXT: ld1 { v4.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #1024 -; CHECK-SD-NEXT: ld1 { v4.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #1032 -; CHECK-SD-NEXT: ld1 { v4.b }[7], [x8] -; CHECK-SD-NEXT: ldr b5, [x9] -; CHECK-SD-NEXT: add x8, sp, #1048 -; CHECK-SD-NEXT: add x9, sp, #728 -; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #1056 -; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #1064 -; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #1072 -; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #1080 -; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #1088 -; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #1096 -; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8] -; CHECK-SD-NEXT: ld1 { v6.b }[1], [x9] -; CHECK-SD-NEXT: add x8, sp, #736 -; CHECK-SD-NEXT: add x9, sp, #784 -; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #744 -; CHECK-SD-NEXT: zip1 v4.2d, v4.2d, v5.2d -; CHECK-SD-NEXT: movi v5.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #752 -; CHECK-SD-NEXT: sdot v19.4s, v4.16b, v1.16b -; CHECK-SD-NEXT: sdot v5.4s, v0.16b, v1.16b -; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #760 -; CHECK-SD-NEXT: ld1 { v6.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #768 -; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #776 -; CHECK-SD-NEXT: ld1 { v6.b }[7], [x8] -; CHECK-SD-NEXT: ldr b7, [x9] -; CHECK-SD-NEXT: add x8, sp, #792 -; CHECK-SD-NEXT: add x9, sp, #856 -; CHECK-SD-NEXT: ld1 { v7.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #800 -; CHECK-SD-NEXT: ld1 { v7.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #808 -; CHECK-SD-NEXT: ld1 { v7.b }[3], [x8] +; CHECK-SD-NEXT: ld1 { v4.b }[11], [x10] +; CHECK-SD-NEXT: add x10, sp, #1072 +; CHECK-SD-NEXT: ld1 { v6.b }[10], [x9] +; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8] +; CHECK-SD-NEXT: add x9, sp, #808 +; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11] +; CHECK-SD-NEXT: add x8, sp, #56 +; CHECK-SD-NEXT: ld1 { v4.b }[12], [x10] +; CHECK-SD-NEXT: add x10, sp, #160 +; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8] +; CHECK-SD-NEXT: ld1 { v6.b }[11], [x9] +; CHECK-SD-NEXT: add x9, sp, #928 +; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10] +; CHECK-SD-NEXT: add x10, sp, #1080 +; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9] ; CHECK-SD-NEXT: add x8, sp, #816 -; CHECK-SD-NEXT: ld1 { v7.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #824 -; CHECK-SD-NEXT: ld1 { v7.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #832 -; CHECK-SD-NEXT: ld1 { v7.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #840 -; CHECK-SD-NEXT: ld1 { v7.b }[7], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[1], [x9] -; CHECK-SD-NEXT: add x8, sp, #864 -; CHECK-SD-NEXT: add x9, sp, #16 -; CHECK-SD-NEXT: ld1 { v16.b }[8], [x9] -; CHECK-SD-NEXT: add x9, sp, #912 -; CHECK-SD-NEXT: ld1 { v17.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #872 -; CHECK-SD-NEXT: zip1 v0.2d, v6.2d, v7.2d -; CHECK-SD-NEXT: ld1 { v16.b }[9], [x10] -; CHECK-SD-NEXT: ld1 { v17.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #880 -; CHECK-SD-NEXT: sdot v19.4s, v0.16b, v1.16b -; CHECK-SD-NEXT: ld1 { v17.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #888 -; CHECK-SD-NEXT: ld1 { v17.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #896 -; CHECK-SD-NEXT: ld1 { v17.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #904 -; CHECK-SD-NEXT: ld1 { v17.b }[7], [x8] -; CHECK-SD-NEXT: ldr b18, [x9] -; CHECK-SD-NEXT: add x8, sp, #920 -; CHECK-SD-NEXT: ld1 { v18.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #32 -; CHECK-SD-NEXT: ld1 { v16.b }[10], [x8] -; CHECK-SD-NEXT: add x8, sp, #928 -; CHECK-SD-NEXT: ld1 { v18.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #40 -; CHECK-SD-NEXT: ld1 { v16.b }[11], [x8] +; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10] +; CHECK-SD-NEXT: add x9, sp, #168 +; CHECK-SD-NEXT: add x10, sp, #176 +; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8] ; CHECK-SD-NEXT: add x8, sp, #936 -; CHECK-SD-NEXT: ld1 { v18.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #48 -; CHECK-SD-NEXT: ld1 { v16.b }[12], [x8] -; CHECK-SD-NEXT: add x8, sp, #944 -; CHECK-SD-NEXT: ld1 { v18.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #56 -; CHECK-SD-NEXT: ld1 { v16.b }[13], [x8] -; CHECK-SD-NEXT: add x8, sp, #952 -; CHECK-SD-NEXT: ld1 { v18.b }[5], [x8] +; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9] +; CHECK-SD-NEXT: add x9, sp, #1088 +; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8] ; CHECK-SD-NEXT: add x8, sp, #64 -; CHECK-SD-NEXT: ld1 { v16.b }[14], [x8] +; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9] +; CHECK-SD-NEXT: add x9, sp, #824 +; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8] +; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9] +; CHECK-SD-NEXT: add x9, sp, #944 +; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10] +; CHECK-SD-NEXT: add x10, sp, #1096 +; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9] +; CHECK-SD-NEXT: add x8, sp, #832 +; CHECK-SD-NEXT: ld1 { v4.b }[15], [x10] +; CHECK-SD-NEXT: add x9, sp, #184 +; CHECK-SD-NEXT: add x10, sp, #72 +; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8] +; CHECK-SD-NEXT: add x8, sp, #952 +; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9] +; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8] +; CHECK-SD-NEXT: add x8, sp, #840 +; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10] +; CHECK-SD-NEXT: sdot v2.4s, v4.16b, v1.16b +; CHECK-SD-NEXT: add x9, sp, #192 +; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8] ; CHECK-SD-NEXT: add x8, sp, #960 -; CHECK-SD-NEXT: ld1 { v18.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #72 -; CHECK-SD-NEXT: ld1 { v16.b }[15], [x8] -; CHECK-SD-NEXT: add x8, sp, #968 -; CHECK-SD-NEXT: ld1 { v18.b }[7], [x8] -; CHECK-SD-NEXT: sdot v5.4s, v16.16b, v1.16b -; CHECK-SD-NEXT: zip1 v0.2d, v17.2d, v18.2d -; CHECK-SD-NEXT: sdot v5.4s, v2.16b, v1.16b -; CHECK-SD-NEXT: sdot v19.4s, v0.16b, v1.16b -; CHECK-SD-NEXT: add v0.4s, v5.4s, v19.4s +; CHECK-SD-NEXT: ld1 { v7.b }[14], [x9] +; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8] +; CHECK-SD-NEXT: sdot v3.4s, v0.16b, v1.16b +; CHECK-SD-NEXT: add x8, sp, #200 +; CHECK-SD-NEXT: add x9, sp, #968 +; CHECK-SD-NEXT: sdot v2.4s, v6.16b, v1.16b +; CHECK-SD-NEXT: ld1 { v7.b }[15], [x8] +; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9] +; CHECK-SD-NEXT: sdot v3.4s, v7.16b, v1.16b +; CHECK-SD-NEXT: sdot v2.4s, v5.16b, v1.16b +; CHECK-SD-NEXT: add v0.4s, v3.4s, v2.4s ; CHECK-SD-NEXT: addv s0, v0.4s ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/nontemporal.ll b/llvm/test/CodeGen/AArch64/nontemporal.ll index f7a87ae..f8ba150 100644 --- a/llvm/test/CodeGen/AArch64/nontemporal.ll +++ b/llvm/test/CodeGen/AArch64/nontemporal.ll @@ -683,43 +683,41 @@ define void @test_stnp_v17f32(<17 x float> %v, ptr %ptr) { ; ; CHECK-BE-LABEL: test_stnp_v17f32: ; CHECK-BE: // %bb.0: // %entry -; CHECK-BE-NEXT: // kill: def $s1 killed $s1 def $q1 -; CHECK-BE-NEXT: // kill: def $s0 killed $s0 def $q0 ; CHECK-BE-NEXT: // kill: def $s4 killed $s4 def $q4 -; CHECK-BE-NEXT: // kill: def $s5 killed $s5 def $q5 -; CHECK-BE-NEXT: add x8, sp, #12 -; CHECK-BE-NEXT: add x9, sp, #20 +; CHECK-BE-NEXT: // kill: def $s0 killed $s0 def $q0 ; CHECK-BE-NEXT: ldr s16, [sp, #36] -; CHECK-BE-NEXT: mov v0.s[1], v1.s[0] -; CHECK-BE-NEXT: ldr s1, [sp, #4] +; CHECK-BE-NEXT: // kill: def $s5 killed $s5 def $q5 +; CHECK-BE-NEXT: // kill: def $s1 killed $s1 def $q1 +; CHECK-BE-NEXT: ldr s17, [sp, #4] +; CHECK-BE-NEXT: add x8, sp, #44 ; CHECK-BE-NEXT: mov v4.s[1], v5.s[0] -; CHECK-BE-NEXT: add x10, sp, #52 +; CHECK-BE-NEXT: mov v0.s[1], v1.s[0] ; CHECK-BE-NEXT: // kill: def $s6 killed $s6 def $q6 ; CHECK-BE-NEXT: // kill: def $s2 killed $s2 def $q2 ; CHECK-BE-NEXT: // kill: def $s7 killed $s7 def $q7 ; CHECK-BE-NEXT: // kill: def $s3 killed $s3 def $q3 -; CHECK-BE-NEXT: ld1 { v1.s }[1], [x8] -; CHECK-BE-NEXT: ldr s5, [x9] -; CHECK-BE-NEXT: add x8, sp, #28 -; CHECK-BE-NEXT: add x9, sp, #44 -; CHECK-BE-NEXT: ld1 { v5.s }[1], [x8] -; CHECK-BE-NEXT: ld1 { v16.s }[1], [x9] -; CHECK-BE-NEXT: ldr s17, [x10] -; CHECK-BE-NEXT: add x8, sp, #60 +; CHECK-BE-NEXT: ldr s1, [sp, #68] +; CHECK-BE-NEXT: ld1 { v16.s }[1], [x8] +; CHECK-BE-NEXT: add x8, sp, #12 +; CHECK-BE-NEXT: ld1 { v17.s }[1], [x8] +; CHECK-BE-NEXT: add x8, sp, #52 +; CHECK-BE-NEXT: str s1, [x0, #64] +; CHECK-BE-NEXT: ld1 { v16.s }[2], [x8] +; CHECK-BE-NEXT: add x8, sp, #20 ; CHECK-BE-NEXT: mov v4.s[2], v6.s[0] ; CHECK-BE-NEXT: mov v0.s[2], v2.s[0] -; CHECK-BE-NEXT: ld1 { v17.s }[1], [x8] -; CHECK-BE-NEXT: ldr s2, [sp, #68] -; CHECK-BE-NEXT: add x8, x0, #32 -; CHECK-BE-NEXT: zip1 v1.2d, v1.2d, v5.2d -; CHECK-BE-NEXT: add x9, x0, #48 -; CHECK-BE-NEXT: str s2, [x0, #64] -; CHECK-BE-NEXT: zip1 v5.2d, v16.2d, v17.2d +; CHECK-BE-NEXT: ld1 { v17.s }[2], [x8] +; CHECK-BE-NEXT: add x8, sp, #60 +; CHECK-BE-NEXT: ld1 { v16.s }[3], [x8] +; CHECK-BE-NEXT: add x8, sp, #28 +; CHECK-BE-NEXT: ld1 { v17.s }[3], [x8] ; CHECK-BE-NEXT: mov v4.s[3], v7.s[0] +; CHECK-BE-NEXT: add x8, x0, #48 ; CHECK-BE-NEXT: mov v0.s[3], v3.s[0] -; CHECK-BE-NEXT: st1 { v1.4s }, [x8] +; CHECK-BE-NEXT: st1 { v16.4s }, [x8] +; CHECK-BE-NEXT: add x8, x0, #32 +; CHECK-BE-NEXT: st1 { v17.4s }, [x8] ; CHECK-BE-NEXT: add x8, x0, #16 -; CHECK-BE-NEXT: st1 { v5.4s }, [x9] ; CHECK-BE-NEXT: st1 { v4.4s }, [x8] ; CHECK-BE-NEXT: st1 { v0.4s }, [x0] ; CHECK-BE-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/ptrauth-isel.ll b/llvm/test/CodeGen/AArch64/ptrauth-isel.ll new file mode 100644 index 0000000..7011b94 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/ptrauth-isel.ll @@ -0,0 +1,269 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5 +; RUN: llc < %s -mtriple arm64e-apple-darwin -verify-machineinstrs -stop-after=finalize-isel -global-isel=0 \ +; RUN: | FileCheck %s --check-prefixes=DAGISEL +; RUN: llc < %s -mtriple arm64e-apple-darwin -verify-machineinstrs -stop-after=finalize-isel -global-isel=1 -global-isel-abort=1 \ +; RUN: | FileCheck %s --check-prefixes=GISEL +; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -verify-machineinstrs -stop-after=finalize-isel -global-isel=0 \ +; RUN: | FileCheck %s --check-prefixes=DAGISEL +; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -verify-machineinstrs -stop-after=finalize-isel -global-isel=1 -global-isel-abort=1 \ +; RUN: | FileCheck %s --check-prefixes=GISEL + +; Check MIR produced by the instruction selector to validate properties that +; cannot be reliably tested by only inspecting the final asm output. + +@discvar = dso_local global i64 0 + +; Make sure the components of blend(addr, imm) and integer constants are +; recognized and passed to PAC pseudo via separate operands to prevent +; substitution of the immediate modifier. +; +; MIR output of the instruction selector is inspected, as it is hard to reliably +; distinguish MOVKXi immediately followed by a pseudo from a standalone pseudo +; instruction carrying address and immediate modifiers in its separate operands +; by only observing the final asm output. + +define i64 @small_imm_disc_optimized(i64 %addr) { + ; DAGISEL-LABEL: name: small_imm_disc_optimized + ; DAGISEL: bb.0.entry: + ; DAGISEL-NEXT: liveins: $x0 + ; DAGISEL-NEXT: {{ $}} + ; DAGISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; DAGISEL-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 42 + ; DAGISEL-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64noip = SUBREG_TO_REG 0, killed [[MOVi32imm]], %subreg.sub_32 + ; DAGISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 42, killed $noreg, implicit-def dead $x16, implicit-def dead $x17 + ; DAGISEL-NEXT: $x0 = COPY [[PAC]] + ; DAGISEL-NEXT: RET_ReallyLR implicit $x0 + ; + ; GISEL-LABEL: name: small_imm_disc_optimized + ; GISEL: bb.1.entry: + ; GISEL-NEXT: liveins: $x0 + ; GISEL-NEXT: {{ $}} + ; GISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; GISEL-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 42 + ; GISEL-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64noip = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32 + ; GISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 42, $noreg, implicit-def dead $x16, implicit-def dead $x17 + ; GISEL-NEXT: $x0 = COPY [[PAC]] + ; GISEL-NEXT: RET_ReallyLR implicit $x0 +entry: + %signed = call i64 @llvm.ptrauth.sign(i64 %addr, i32 2, i64 42) + ret i64 %signed +} + +; Without optimization, MOVi64imm may be used for small i64 constants as well. +define i64 @small_imm_disc_non_optimized(i64 %addr) noinline optnone { + ; DAGISEL-LABEL: name: small_imm_disc_non_optimized + ; DAGISEL: bb.0.entry: + ; DAGISEL-NEXT: liveins: $x0 + ; DAGISEL-NEXT: {{ $}} + ; DAGISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; DAGISEL-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY killed [[COPY]] + ; DAGISEL-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 42 + ; DAGISEL-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64noip = SUBREG_TO_REG 0, killed [[MOVi32imm]], %subreg.sub_32 + ; DAGISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY1]], 2, 42, killed $noreg, implicit-def dead $x16, implicit-def dead $x17 + ; DAGISEL-NEXT: [[COPY2:%[0-9]+]]:gpr64all = COPY [[PAC]] + ; DAGISEL-NEXT: $x0 = COPY [[COPY2]] + ; DAGISEL-NEXT: RET_ReallyLR implicit $x0 + ; + ; GISEL-LABEL: name: small_imm_disc_non_optimized + ; GISEL: bb.1.entry: + ; GISEL-NEXT: liveins: $x0 + ; GISEL-NEXT: {{ $}} + ; GISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; GISEL-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64noip = MOVi64imm 42 + ; GISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 42, $noreg, implicit-def dead $x16, implicit-def dead $x17 + ; GISEL-NEXT: $x0 = COPY [[PAC]] + ; GISEL-NEXT: RET_ReallyLR implicit $x0 +entry: + %signed = call i64 @llvm.ptrauth.sign(i64 %addr, i32 2, i64 42) + ret i64 %signed +} + +define i64 @large_imm_disc_wreg(i64 %addr) { + ; DAGISEL-LABEL: name: large_imm_disc_wreg + ; DAGISEL: bb.0.entry: + ; DAGISEL-NEXT: liveins: $x0 + ; DAGISEL-NEXT: {{ $}} + ; DAGISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; DAGISEL-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 12345678 + ; DAGISEL-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64noip = SUBREG_TO_REG 0, killed [[MOVi32imm]], %subreg.sub_32 + ; DAGISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 0, killed [[SUBREG_TO_REG]], implicit-def dead $x16, implicit-def dead $x17 + ; DAGISEL-NEXT: $x0 = COPY [[PAC]] + ; DAGISEL-NEXT: RET_ReallyLR implicit $x0 + ; + ; GISEL-LABEL: name: large_imm_disc_wreg + ; GISEL: bb.1.entry: + ; GISEL-NEXT: liveins: $x0 + ; GISEL-NEXT: {{ $}} + ; GISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; GISEL-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 12345678 + ; GISEL-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64noip = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32 + ; GISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 0, [[SUBREG_TO_REG]], implicit-def dead $x16, implicit-def dead $x17 + ; GISEL-NEXT: $x0 = COPY [[PAC]] + ; GISEL-NEXT: RET_ReallyLR implicit $x0 +entry: + %signed = call i64 @llvm.ptrauth.sign(i64 %addr, i32 2, i64 12345678) + ret i64 %signed +} + +define i64 @large_imm_disc_xreg(i64 %addr) { + ; DAGISEL-LABEL: name: large_imm_disc_xreg + ; DAGISEL: bb.0.entry: + ; DAGISEL-NEXT: liveins: $x0 + ; DAGISEL-NEXT: {{ $}} + ; DAGISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; DAGISEL-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64noip = MOVi64imm 123456789012345 + ; DAGISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 0, killed [[MOVi64imm]], implicit-def dead $x16, implicit-def dead $x17 + ; DAGISEL-NEXT: $x0 = COPY [[PAC]] + ; DAGISEL-NEXT: RET_ReallyLR implicit $x0 + ; + ; GISEL-LABEL: name: large_imm_disc_xreg + ; GISEL: bb.1.entry: + ; GISEL-NEXT: liveins: $x0 + ; GISEL-NEXT: {{ $}} + ; GISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; GISEL-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64noip = MOVi64imm 123456789012345 + ; GISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 0, [[MOVi64imm]], implicit-def dead $x16, implicit-def dead $x17 + ; GISEL-NEXT: $x0 = COPY [[PAC]] + ; GISEL-NEXT: RET_ReallyLR implicit $x0 +entry: + %signed = call i64 @llvm.ptrauth.sign(i64 %addr, i32 2, i64 123456789012345) + ret i64 %signed +} + +; Make sure blend() is lowered as expected when optimization is disabled. +define i64 @blended_disc_non_optimized(i64 %addr, i64 %addrdisc) noinline optnone { + ; DAGISEL-LABEL: name: blended_disc_non_optimized + ; DAGISEL: bb.0.entry: + ; DAGISEL-NEXT: liveins: $x0, $x1 + ; DAGISEL-NEXT: {{ $}} + ; DAGISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x1 + ; DAGISEL-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x0 + ; DAGISEL-NEXT: [[COPY2:%[0-9]+]]:gpr64 = COPY killed [[COPY1]] + ; DAGISEL-NEXT: [[COPY3:%[0-9]+]]:gpr64 = COPY killed [[COPY]] + ; DAGISEL-NEXT: [[MOVKXi:%[0-9]+]]:gpr64 = MOVKXi [[COPY3]], 42, 48 + ; DAGISEL-NEXT: [[COPY4:%[0-9]+]]:gpr64noip = COPY [[MOVKXi]] + ; DAGISEL-NEXT: [[COPY5:%[0-9]+]]:gpr64noip = COPY [[COPY3]] + ; DAGISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY2]], 2, 42, [[COPY5]], implicit-def dead $x16, implicit-def dead $x17 + ; DAGISEL-NEXT: [[COPY6:%[0-9]+]]:gpr64all = COPY [[PAC]] + ; DAGISEL-NEXT: $x0 = COPY [[COPY6]] + ; DAGISEL-NEXT: RET_ReallyLR implicit $x0 + ; + ; GISEL-LABEL: name: blended_disc_non_optimized + ; GISEL: bb.1.entry: + ; GISEL-NEXT: liveins: $x0, $x1 + ; GISEL-NEXT: {{ $}} + ; GISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; GISEL-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; GISEL-NEXT: [[MOVKXi:%[0-9]+]]:gpr64noip = MOVKXi [[COPY1]], 42, 48 + ; GISEL-NEXT: [[COPY2:%[0-9]+]]:gpr64noip = COPY [[COPY1]] + ; GISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 42, [[COPY2]], implicit-def dead $x16, implicit-def dead $x17 + ; GISEL-NEXT: $x0 = COPY [[PAC]] + ; GISEL-NEXT: RET_ReallyLR implicit $x0 +entry: + %disc = call i64 @llvm.ptrauth.blend(i64 %addrdisc, i64 42) + %signed = call i64 @llvm.ptrauth.sign(i64 %addr, i32 2, i64 %disc) + ret i64 %signed +} + +define i64 @blend_and_sign_same_bb(i64 %addr) { + ; DAGISEL-LABEL: name: blend_and_sign_same_bb + ; DAGISEL: bb.0.entry: + ; DAGISEL-NEXT: liveins: $x0 + ; DAGISEL-NEXT: {{ $}} + ; DAGISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; DAGISEL-NEXT: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) @discvar + ; DAGISEL-NEXT: [[LDRXui:%[0-9]+]]:gpr64 = LDRXui killed [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) @discvar :: (dereferenceable load (s64) from @discvar) + ; DAGISEL-NEXT: [[MOVKXi:%[0-9]+]]:gpr64noip = MOVKXi [[LDRXui]], 42, 48 + ; DAGISEL-NEXT: [[COPY1:%[0-9]+]]:gpr64noip = COPY [[LDRXui]] + ; DAGISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 42, killed [[COPY1]], implicit-def dead $x16, implicit-def dead $x17 + ; DAGISEL-NEXT: $x0 = COPY [[PAC]] + ; DAGISEL-NEXT: RET_ReallyLR implicit $x0 + ; + ; GISEL-LABEL: name: blend_and_sign_same_bb + ; GISEL: bb.1.entry: + ; GISEL-NEXT: liveins: $x0 + ; GISEL-NEXT: {{ $}} + ; GISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; GISEL-NEXT: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) @discvar + ; GISEL-NEXT: [[LDRXui:%[0-9]+]]:gpr64 = LDRXui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) @discvar :: (dereferenceable load (s64) from @discvar) + ; GISEL-NEXT: [[MOVKXi:%[0-9]+]]:gpr64noip = MOVKXi [[LDRXui]], 42, 48 + ; GISEL-NEXT: [[COPY1:%[0-9]+]]:gpr64noip = COPY [[LDRXui]] + ; GISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 42, [[COPY1]], implicit-def dead $x16, implicit-def dead $x17 + ; GISEL-NEXT: $x0 = COPY [[PAC]] + ; GISEL-NEXT: RET_ReallyLR implicit $x0 +entry: + %addrdisc = load i64, ptr @discvar + %disc = call i64 @llvm.ptrauth.blend(i64 %addrdisc, i64 42) + %signed = call i64 @llvm.ptrauth.sign(i64 %addr, i32 2, i64 %disc) + ret i64 %signed +} + +; In the below test cases both %addrdisc and %disc are computed (i.e. they are +; neither global addresses, nor function arguments) in a different basic block, +; making them harder to express via ISD::PtrAuthGlobalAddress. + +define i64 @blend_and_sign_different_bbs(i64 %addr, i64 %cond) { + ; DAGISEL-LABEL: name: blend_and_sign_different_bbs + ; DAGISEL: bb.0.entry: + ; DAGISEL-NEXT: successors: %bb.1(0x50000000), %bb.2(0x30000000) + ; DAGISEL-NEXT: liveins: $x0, $x1 + ; DAGISEL-NEXT: {{ $}} + ; DAGISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x1 + ; DAGISEL-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x0 + ; DAGISEL-NEXT: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) @discvar + ; DAGISEL-NEXT: [[LDRXui:%[0-9]+]]:gpr64 = LDRXui killed [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) @discvar :: (dereferenceable load (s64) from @discvar) + ; DAGISEL-NEXT: [[MOVKXi:%[0-9]+]]:gpr64 = MOVKXi [[LDRXui]], 42, 48 + ; DAGISEL-NEXT: [[COPY2:%[0-9]+]]:gpr64noip = COPY [[MOVKXi]] + ; DAGISEL-NEXT: CBZX [[COPY]], %bb.2 + ; DAGISEL-NEXT: B %bb.1 + ; DAGISEL-NEXT: {{ $}} + ; DAGISEL-NEXT: bb.1.next: + ; DAGISEL-NEXT: successors: %bb.2(0x80000000) + ; DAGISEL-NEXT: {{ $}} + ; DAGISEL-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY [[COPY2]] + ; DAGISEL-NEXT: INLINEASM &nop, 1 /* sideeffect attdialect */, 3866633 /* reguse:GPR64common */, [[COPY3]] + ; DAGISEL-NEXT: {{ $}} + ; DAGISEL-NEXT: bb.2.exit: + ; DAGISEL-NEXT: [[COPY4:%[0-9]+]]:gpr64noip = COPY [[LDRXui]] + ; DAGISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY1]], 2, 42, [[COPY4]], implicit-def dead $x16, implicit-def dead $x17 + ; DAGISEL-NEXT: $x0 = COPY [[PAC]] + ; DAGISEL-NEXT: RET_ReallyLR implicit $x0 + ; + ; GISEL-LABEL: name: blend_and_sign_different_bbs + ; GISEL: bb.1.entry: + ; GISEL-NEXT: successors: %bb.2(0x50000000), %bb.3(0x30000000) + ; GISEL-NEXT: liveins: $x0, $x1 + ; GISEL-NEXT: {{ $}} + ; GISEL-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; GISEL-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; GISEL-NEXT: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) @discvar + ; GISEL-NEXT: [[LDRXui:%[0-9]+]]:gpr64 = LDRXui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) @discvar :: (dereferenceable load (s64) from @discvar) + ; GISEL-NEXT: [[MOVKXi:%[0-9]+]]:gpr64noip = MOVKXi [[LDRXui]], 42, 48 + ; GISEL-NEXT: CBZX [[COPY1]], %bb.3 + ; GISEL-NEXT: B %bb.2 + ; GISEL-NEXT: {{ $}} + ; GISEL-NEXT: bb.2.next: + ; GISEL-NEXT: successors: %bb.3(0x80000000) + ; GISEL-NEXT: {{ $}} + ; GISEL-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY [[MOVKXi]] + ; GISEL-NEXT: INLINEASM &nop, 1 /* sideeffect attdialect */, 3866633 /* reguse:GPR64common */, [[COPY2]] + ; GISEL-NEXT: {{ $}} + ; GISEL-NEXT: bb.3.exit: + ; GISEL-NEXT: [[COPY3:%[0-9]+]]:gpr64noip = COPY [[LDRXui]] + ; GISEL-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 42, [[COPY3]], implicit-def dead $x16, implicit-def dead $x17 + ; GISEL-NEXT: $x0 = COPY [[PAC]] + ; GISEL-NEXT: RET_ReallyLR implicit $x0 +entry: + %addrdisc = load i64, ptr @discvar + %disc = call i64 @llvm.ptrauth.blend(i64 %addrdisc, i64 42) + %cond.b = icmp ne i64 %cond, 0 + br i1 %cond.b, label %next, label %exit + +next: + call void asm sideeffect "nop", "r"(i64 %disc) + br label %exit + +exit: + %signed = call i64 @llvm.ptrauth.sign(i64 %addr, i32 2, i64 %disc) + ret i64 %signed +} diff --git a/llvm/test/CodeGen/AArch64/ptrauth-isel.mir b/llvm/test/CodeGen/AArch64/ptrauth-isel.mir new file mode 100644 index 0000000..1a15588 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/ptrauth-isel.mir @@ -0,0 +1,205 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5 +# RUN: llc -o - %s -mtriple arm64e-apple-darwin -verify-machineinstrs \ +# RUN: -stop-after=finalize-isel -start-before=finalize-isel | FileCheck %s +# RUN: llc -o - %s -mtriple aarch64-linux-gnu -mattr=+pauth -verify-machineinstrs \ +# RUN: -stop-after=finalize-isel -start-before=finalize-isel | FileCheck %s + +# This MIR-based test contains several test cases that are hard to implement +# via an LLVM IR input. Most other test cases are in ptrauth-isel.ll file. + +--- | + @globalvar = dso_local global i64 0 + + define i64 @movk_correct_blend(i64 %a, i64 %b) { + entry: + ret i64 0 + } + + define i64 @movk_wrong_shift_amount(i64 %a, i64 %b) { + entry: + ret i64 0 + } + + define i64 @movk_non_immediate_operand(i64 %a, i64 %b) { + entry: + ret i64 0 + } + + define i64 @movi64imm_immediate_operand(i64 %a) { + entry: + ret i64 0 + } + + define i64 @movi64imm_non_immediate_operand(i64 %a) { + entry: + ret i64 0 + } + + define i64 @movi32imm_immediate_operand(i64 %a) { + entry: + ret i64 0 + } + + define i64 @movi32imm_non_immediate_operand(i64 %a) { + entry: + ret i64 0 + } +... +--- +name: movk_correct_blend +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: movk_correct_blend + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[MOVKXi:%[0-9]+]]:gpr64noip = MOVKXi [[COPY1]], 42, 48 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64noip = COPY [[COPY1]] + ; CHECK-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 42, killed [[COPY2]], implicit-def dead $x16, implicit-def dead $x17 + ; CHECK-NEXT: $x0 = COPY [[PAC]] + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:gpr64 = COPY $x0 + %1:gpr64 = COPY $x1 + %2:gpr64noip = MOVKXi %1, 42, 48 + %3:gpr64 = PAC %0, 2, 0, killed %2, implicit-def dead $x16, implicit-def dead $x17 + $x0 = COPY %3 + RET_ReallyLR implicit $x0 +... +--- +name: movk_wrong_shift_amount +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: movk_wrong_shift_amount + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[MOVKXi:%[0-9]+]]:gpr64noip = MOVKXi [[COPY1]], 42, 0 + ; CHECK-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 0, killed [[MOVKXi]], implicit-def dead $x16, implicit-def dead $x17 + ; CHECK-NEXT: $x0 = COPY [[PAC]] + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:gpr64 = COPY $x0 + %1:gpr64 = COPY $x1 + %2:gpr64noip = MOVKXi %1, 42, 0 + %3:gpr64 = PAC %0, 2, 0, killed %2, implicit-def dead $x16, implicit-def dead $x17 + $x0 = COPY %3 + RET_ReallyLR implicit $x0 +... +--- +name: movk_non_immediate_operand +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: movk_non_immediate_operand + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[MOVKXi:%[0-9]+]]:gpr64noip = MOVKXi [[COPY1]], target-flags(aarch64-pageoff, aarch64-nc) @globalvar, 48 + ; CHECK-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 0, killed [[MOVKXi]], implicit-def dead $x16, implicit-def dead $x17 + ; CHECK-NEXT: $x0 = COPY [[PAC]] + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:gpr64 = COPY $x0 + %1:gpr64 = COPY $x1 + %2:gpr64noip = MOVKXi %1, target-flags(aarch64-pageoff, aarch64-nc) @globalvar, 48 + %3:gpr64 = PAC %0, 2, 0, killed %2, implicit-def dead $x16, implicit-def dead $x17 + $x0 = COPY %3 + RET_ReallyLR implicit $x0 +... +--- +name: movi64imm_immediate_operand +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: movi64imm_immediate_operand + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64noip = MOVi64imm 42 + ; CHECK-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 42, killed $noreg, implicit-def dead $x16, implicit-def dead $x17 + ; CHECK-NEXT: $x0 = COPY [[PAC]] + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:gpr64 = COPY $x0 + %1:gpr64noip = MOVi64imm 42 + %2:gpr64 = PAC %0, 2, 0, killed %1, implicit-def dead $x16, implicit-def dead $x17 + $x0 = COPY %2 + RET_ReallyLR implicit $x0 +... +--- +name: movi64imm_non_immediate_operand +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: movi64imm_non_immediate_operand + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64noip = MOVi64imm target-flags(aarch64-pageoff, aarch64-nc) @globalvar + ; CHECK-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 0, killed [[MOVi64imm]], implicit-def dead $x16, implicit-def dead $x17 + ; CHECK-NEXT: $x0 = COPY [[PAC]] + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:gpr64 = COPY $x0 + %1:gpr64noip = MOVi64imm target-flags(aarch64-pageoff, aarch64-nc) @globalvar + %2:gpr64 = PAC %0, 2, 0, killed %1, implicit-def dead $x16, implicit-def dead $x17 + $x0 = COPY %2 + RET_ReallyLR implicit $x0 +... +--- +name: movi32imm_immediate_operand +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: movi32imm_immediate_operand + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 42 + ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64noip = SUBREG_TO_REG 0, killed [[MOVi32imm]], %subreg.sub_32 + ; CHECK-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 42, killed $noreg, implicit-def dead $x16, implicit-def dead $x17 + ; CHECK-NEXT: $x0 = COPY [[PAC]] + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:gpr64 = COPY $x0 + %1:gpr32 = MOVi32imm 42 + %2:gpr64noip = SUBREG_TO_REG 0, killed %1, %subreg.sub_32 + %3:gpr64 = PAC %0, 2, 0, killed %2, implicit-def dead $x16, implicit-def dead $x17 + $x0 = COPY %3 + RET_ReallyLR implicit $x0 +... +--- +name: movi32imm_non_immediate_operand +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: movi32imm_non_immediate_operand + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm target-flags(aarch64-pageoff, aarch64-nc) @globalvar + ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64noip = SUBREG_TO_REG 0, killed [[MOVi32imm]], %subreg.sub_32 + ; CHECK-NEXT: [[PAC:%[0-9]+]]:gpr64 = PAC [[COPY]], 2, 0, killed [[SUBREG_TO_REG]], implicit-def dead $x16, implicit-def dead $x17 + ; CHECK-NEXT: $x0 = COPY [[PAC]] + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:gpr64 = COPY $x0 + %1:gpr32 = MOVi32imm target-flags(aarch64-pageoff, aarch64-nc) @globalvar + %2:gpr64noip = SUBREG_TO_REG 0, killed %1, %subreg.sub_32 + %3:gpr64 = PAC %0, 2, 0, killed %2, implicit-def dead $x16, implicit-def dead $x17 + $x0 = COPY %3 + RET_ReallyLR implicit $x0 +... diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll index 8192d4a..0e132f1 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll @@ -70,12 +70,12 @@ define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc: @@ -85,12 +85,12 @@ define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2) @@ -153,12 +153,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrsp ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc: @@ -168,12 +168,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrsp ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2) @@ -236,12 +236,12 @@ define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> % ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc: @@ -251,12 +251,12 @@ define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> % ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2) @@ -319,12 +319,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr add ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc: @@ -334,12 +334,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr add ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2) @@ -402,12 +402,12 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc: @@ -417,12 +417,12 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2) @@ -485,12 +485,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrsp ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc: @@ -500,12 +500,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrsp ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2) @@ -568,12 +568,12 @@ define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> % ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc: @@ -583,12 +583,12 @@ define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> % ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2) @@ -651,12 +651,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr add ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc: @@ -666,12 +666,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr add ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2) @@ -734,12 +734,12 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc: @@ -749,12 +749,12 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2) @@ -817,12 +817,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc: @@ -832,12 +832,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2) @@ -900,12 +900,12 @@ define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> % ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc: @@ -915,12 +915,12 @@ define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> % ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2) @@ -983,12 +983,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr add ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc: @@ -998,12 +998,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr add ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2) diff --git a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir index 1a457c9..9241a23 100644 --- a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir +++ b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir @@ -38,20 +38,20 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4) ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1) - ; CHECK-NEXT: renamable $vgpr4 = V_MOV_B32_e32 1065353216, implicit $exec - ; CHECK-NEXT: renamable $vgpr5 = V_MOV_B32_e32 0, implicit $exec - ; CHECK-NEXT: renamable $vgpr6 = V_MOV_B32_e32 1073741824, implicit $exec + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1065353216, implicit $exec + ; CHECK-NEXT: renamable $vgpr4 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $vgpr1 = V_MOV_B32_e32 1073741824, implicit $exec ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3 - ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec - ; CHECK-NEXT: renamable $vgpr1 = COPY renamable $agpr1 - ; CHECK-NEXT: renamable $vgpr0 = COPY renamable $agpr0 - ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec - ; CHECK-NEXT: renamable $vgpr3 = COPY renamable $agpr1 - ; CHECK-NEXT: renamable $vgpr2 = COPY killed renamable $agpr0 - ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3 - ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 killed $vgpr4, killed $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr0, $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr6 = COPY renamable $agpr1 + ; CHECK-NEXT: renamable $vgpr5 = COPY renamable $agpr0 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr0, $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr8 = COPY renamable $agpr1 + ; CHECK-NEXT: renamable $vgpr7 = COPY killed renamable $agpr0 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr5_vgpr6_vgpr7_vgpr8 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3 - ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr5, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr4, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1) ; CHECK-NEXT: S_ENDPGM 0 early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4) renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1) diff --git a/llvm/test/CodeGen/AMDGPU/bf16-math.ll b/llvm/test/CodeGen/AMDGPU/bf16-math.ll index 1adf542..9979e83 100644 --- a/llvm/test/CodeGen/AMDGPU/bf16-math.ll +++ b/llvm/test/CodeGen/AMDGPU/bf16-math.ll @@ -323,6 +323,146 @@ define amdgpu_ps void @v_test_max_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> ret void } +define amdgpu_ps bfloat @test_clamp_bf16(bfloat %src) { +; GCN-LABEL: test_clamp_bf16: +; GCN: ; %bb.0: +; GCN-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp +; GCN-NEXT: ; return to shader part epilog + %max = call bfloat @llvm.maxnum.bf16(bfloat %src, bfloat 0.0) + %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0) + ret bfloat %clamp +} + +define amdgpu_ps bfloat @test_clamp_bf16_s(bfloat inreg %src) { +; GCN-LABEL: test_clamp_bf16_s: +; GCN: ; %bb.0: +; GCN-NEXT: v_pk_max_num_bf16 v0, s0, s0 clamp +; GCN-NEXT: ; return to shader part epilog + %max = call bfloat @llvm.maxnum.bf16(bfloat %src, bfloat 0.0) + %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0) + ret bfloat %clamp +} + +define amdgpu_ps float @test_clamp_v2bf16(<2 x bfloat> %src) { +; GCN-LABEL: test_clamp_v2bf16: +; GCN: ; %bb.0: +; GCN-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp +; GCN-NEXT: ; return to shader part epilog + %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %src, <2 x bfloat> <bfloat 0.0, bfloat 0.0>) + %clamp = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %max, <2 x bfloat> <bfloat 1.0, bfloat 1.0>) + %ret = bitcast <2 x bfloat> %clamp to float + ret float %ret +} + +define amdgpu_ps float @test_clamp_v2bf16_s(<2 x bfloat> inreg %src) { +; GCN-LABEL: test_clamp_v2bf16_s: +; GCN: ; %bb.0: +; GCN-NEXT: v_pk_max_num_bf16 v0, s0, s0 clamp +; GCN-NEXT: ; return to shader part epilog + %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %src, <2 x bfloat> <bfloat 0.0, bfloat 0.0>) + %clamp = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %max, <2 x bfloat> <bfloat 1.0, bfloat 1.0>) + %ret = bitcast <2 x bfloat> %clamp to float + ret float %ret +} + +define amdgpu_ps bfloat @test_clamp_bf16_folding(bfloat %src) { +; GCN-LABEL: test_clamp_bf16_folding: +; GCN: ; %bb.0: +; GCN-NEXT: v_exp_bf16_e32 v0, v0 +; GCN-NEXT: v_nop +; GCN-NEXT: s_delay_alu instid0(TRANS32_DEP_1) +; GCN-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp +; GCN-NEXT: ; return to shader part epilog + %exp = call bfloat @llvm.exp2.bf16(bfloat %src) + %max = call bfloat @llvm.maxnum.bf16(bfloat %exp, bfloat 0.0) + %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0) + ret bfloat %clamp +} + +define amdgpu_ps float @test_clamp_v2bf16_folding(<2 x bfloat> %src0, <2 x bfloat> %src1) { +; GCN-LABEL: test_clamp_v2bf16_folding: +; GCN: ; %bb.0: +; GCN-NEXT: v_pk_mul_bf16 v0, v0, v1 +; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GCN-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp +; GCN-NEXT: ; return to shader part epilog + %mul = fmul <2 x bfloat> %src0, %src1 + %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %mul, <2 x bfloat> <bfloat 0.0, bfloat 0.0>) + %clamp = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %max, <2 x bfloat> <bfloat 1.0, bfloat 1.0>) + %ret = bitcast <2 x bfloat> %clamp to float + ret float %ret +} + +define amdgpu_ps void @v_test_mul_add_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) { +; GCN-LABEL: v_test_mul_add_v2bf16_vvv: +; GCN: ; %bb.0: +; GCN-NEXT: v_pk_mul_bf16 v2, v2, v3 +; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GCN-NEXT: v_pk_add_bf16 v2, v2, v4 +; GCN-NEXT: global_store_b32 v[0:1], v2, off +; GCN-NEXT: s_endpgm + %mul = fmul contract <2 x bfloat> %a, %b + %add = fadd contract <2 x bfloat> %mul, %c + store <2 x bfloat> %add, ptr addrspace(1) %out + ret void +} + +define amdgpu_ps void @v_test_mul_add_v2bf16_vss(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) { +; GCN-LABEL: v_test_mul_add_v2bf16_vss: +; GCN: ; %bb.0: +; GCN-NEXT: v_pk_mul_bf16 v2, v2, s0 +; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GCN-NEXT: v_pk_add_bf16 v2, v2, s1 +; GCN-NEXT: global_store_b32 v[0:1], v2, off +; GCN-NEXT: s_endpgm + %mul = fmul contract <2 x bfloat> %a, %b + %add = fadd contract <2 x bfloat> %mul, %c + store <2 x bfloat> %add, ptr addrspace(1) %out + ret void +} + +define amdgpu_ps void @v_test_mul_add_v2bf16_sss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) { +; GCN-LABEL: v_test_mul_add_v2bf16_sss: +; GCN: ; %bb.0: +; GCN-NEXT: v_pk_mul_bf16 v2, s0, s1 +; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GCN-NEXT: v_pk_add_bf16 v2, v2, s2 +; GCN-NEXT: global_store_b32 v[0:1], v2, off +; GCN-NEXT: s_endpgm + %mul = fmul contract <2 x bfloat> %a, %b + %add = fadd contract <2 x bfloat> %mul, %c + store <2 x bfloat> %add, ptr addrspace(1) %out + ret void +} + +define amdgpu_ps void @v_test_mul_add_v2bf16_vsc(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) { +; GCN-LABEL: v_test_mul_add_v2bf16_vsc: +; GCN: ; %bb.0: +; GCN-NEXT: v_pk_mul_bf16 v2, v2, s0 +; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GCN-NEXT: v_pk_add_bf16 v2, v2, 0.5 op_sel_hi:[1,0] +; GCN-NEXT: global_store_b32 v[0:1], v2, off +; GCN-NEXT: s_endpgm + %mul = fmul contract <2 x bfloat> %a, %b + %add = fadd contract <2 x bfloat> %mul, <bfloat 0.5, bfloat 0.5> + store <2 x bfloat> %add, ptr addrspace(1) %out + ret void +} + +define amdgpu_ps void @v_test_mul_add_v2bf16_vll(ptr addrspace(1) %out, <2 x bfloat> %a) { +; GCN-LABEL: v_test_mul_add_v2bf16_vll: +; GCN: ; %bb.0: +; GCN-NEXT: v_pk_mul_bf16 v2, 0x42c83f80, v2 +; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GCN-NEXT: v_pk_add_bf16 v2, 0x43484000, v2 +; GCN-NEXT: global_store_b32 v[0:1], v2, off +; GCN-NEXT: s_endpgm + %mul = fmul contract <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0> + %add = fadd contract <2 x bfloat> %mul, <bfloat 2.0, bfloat 200.0> + store <2 x bfloat> %add, ptr addrspace(1) %out + ret void +} + define amdgpu_ps void @v_test_fma_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) { ; GCN-LABEL: v_test_fma_v2bf16_vvv: ; GCN: ; %bb.0: @@ -426,6 +566,8 @@ define amdgpu_ps void @llvm_exp2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src ret void } +declare bfloat @llvm.minnum.bf16(bfloat, bfloat) +declare bfloat @llvm.maxnum.bf16(bfloat, bfloat) declare <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) declare <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) declare <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>) diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll index 7859fcdf..52e697c 100644 --- a/llvm/test/CodeGen/AMDGPU/bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/bf16.ll @@ -468,15 +468,28 @@ define <16 x bfloat> @v_load_global_v16bf16(ptr addrspace(1) %ptr) { ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] ; -; GFX9-LABEL: v_load_global_v16bf16: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v9, v1 -; GFX9-NEXT: v_mov_b32_e32 v8, v0 -; GFX9-NEXT: global_load_dwordx4 v[0:3], v[8:9], off -; GFX9-NEXT: global_load_dwordx4 v[4:7], v[8:9], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GFX900-LABEL: v_load_global_v16bf16: +; GFX900: ; %bb.0: +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v9, v1 +; GFX900-NEXT: v_mov_b32_e32 v8, v0 +; GFX900-NEXT: global_load_dwordx4 v[0:3], v[8:9], off +; GFX900-NEXT: global_load_dwordx4 v[4:7], v[8:9], off offset:16 +; GFX900-NEXT: s_waitcnt vmcnt(0) +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX950-LABEL: v_load_global_v16bf16: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: global_load_dwordx4 v[8:11], v[0:1], off +; GFX950-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:16 +; GFX950-NEXT: s_waitcnt vmcnt(1) +; GFX950-NEXT: v_mov_b32_e32 v0, v8 +; GFX950-NEXT: v_mov_b32_e32 v1, v9 +; GFX950-NEXT: v_mov_b32_e32 v2, v10 +; GFX950-NEXT: v_mov_b32_e32 v3, v11 +; GFX950-NEXT: s_waitcnt vmcnt(0) +; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_load_global_v16bf16: ; GFX10: ; %bb.0: @@ -619,17 +632,32 @@ define <32 x bfloat> @v_load_global_v32bf16(ptr addrspace(1) %ptr) { ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] ; -; GFX9-LABEL: v_load_global_v32bf16: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v17, v1 -; GFX9-NEXT: v_mov_b32_e32 v16, v0 -; GFX9-NEXT: global_load_dwordx4 v[0:3], v[16:17], off -; GFX9-NEXT: global_load_dwordx4 v[4:7], v[16:17], off offset:16 -; GFX9-NEXT: global_load_dwordx4 v[8:11], v[16:17], off offset:32 -; GFX9-NEXT: global_load_dwordx4 v[12:15], v[16:17], off offset:48 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GFX900-LABEL: v_load_global_v32bf16: +; GFX900: ; %bb.0: +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v17, v1 +; GFX900-NEXT: v_mov_b32_e32 v16, v0 +; GFX900-NEXT: global_load_dwordx4 v[0:3], v[16:17], off +; GFX900-NEXT: global_load_dwordx4 v[4:7], v[16:17], off offset:16 +; GFX900-NEXT: global_load_dwordx4 v[8:11], v[16:17], off offset:32 +; GFX900-NEXT: global_load_dwordx4 v[12:15], v[16:17], off offset:48 +; GFX900-NEXT: s_waitcnt vmcnt(0) +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX950-LABEL: v_load_global_v32bf16: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: global_load_dwordx4 v[16:19], v[0:1], off +; GFX950-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:16 +; GFX950-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:32 +; GFX950-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:48 +; GFX950-NEXT: s_waitcnt vmcnt(3) +; GFX950-NEXT: v_mov_b32_e32 v0, v16 +; GFX950-NEXT: v_mov_b32_e32 v1, v17 +; GFX950-NEXT: v_mov_b32_e32 v2, v18 +; GFX950-NEXT: v_mov_b32_e32 v3, v19 +; GFX950-NEXT: s_waitcnt vmcnt(0) +; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_load_global_v32bf16: ; GFX10: ; %bb.0: @@ -877,22 +905,41 @@ define <64 x bfloat> @v_load_global_v64bf16(ptr addrspace(1) %ptr) { ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] ; -; GFX9-LABEL: v_load_global_v64bf16: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v29, v1 -; GFX9-NEXT: v_mov_b32_e32 v28, v0 -; GFX9-NEXT: global_load_dwordx4 v[0:3], v[28:29], off -; GFX9-NEXT: global_load_dwordx4 v[4:7], v[28:29], off offset:16 -; GFX9-NEXT: global_load_dwordx4 v[8:11], v[28:29], off offset:32 -; GFX9-NEXT: global_load_dwordx4 v[12:15], v[28:29], off offset:48 -; GFX9-NEXT: global_load_dwordx4 v[16:19], v[28:29], off offset:64 -; GFX9-NEXT: global_load_dwordx4 v[20:23], v[28:29], off offset:80 -; GFX9-NEXT: global_load_dwordx4 v[24:27], v[28:29], off offset:96 -; GFX9-NEXT: s_nop 0 -; GFX9-NEXT: global_load_dwordx4 v[28:31], v[28:29], off offset:112 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GFX900-LABEL: v_load_global_v64bf16: +; GFX900: ; %bb.0: +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v29, v1 +; GFX900-NEXT: v_mov_b32_e32 v28, v0 +; GFX900-NEXT: global_load_dwordx4 v[0:3], v[28:29], off +; GFX900-NEXT: global_load_dwordx4 v[4:7], v[28:29], off offset:16 +; GFX900-NEXT: global_load_dwordx4 v[8:11], v[28:29], off offset:32 +; GFX900-NEXT: global_load_dwordx4 v[12:15], v[28:29], off offset:48 +; GFX900-NEXT: global_load_dwordx4 v[16:19], v[28:29], off offset:64 +; GFX900-NEXT: global_load_dwordx4 v[20:23], v[28:29], off offset:80 +; GFX900-NEXT: global_load_dwordx4 v[24:27], v[28:29], off offset:96 +; GFX900-NEXT: s_nop 0 +; GFX900-NEXT: global_load_dwordx4 v[28:31], v[28:29], off offset:112 +; GFX900-NEXT: s_waitcnt vmcnt(0) +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX950-LABEL: v_load_global_v64bf16: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: global_load_dwordx4 v[32:35], v[0:1], off +; GFX950-NEXT: global_load_dwordx4 v[4:7], v[0:1], off offset:16 +; GFX950-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:32 +; GFX950-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:48 +; GFX950-NEXT: global_load_dwordx4 v[16:19], v[0:1], off offset:64 +; GFX950-NEXT: global_load_dwordx4 v[20:23], v[0:1], off offset:80 +; GFX950-NEXT: global_load_dwordx4 v[24:27], v[0:1], off offset:96 +; GFX950-NEXT: global_load_dwordx4 v[28:31], v[0:1], off offset:112 +; GFX950-NEXT: s_waitcnt vmcnt(7) +; GFX950-NEXT: v_mov_b32_e32 v0, v32 +; GFX950-NEXT: v_mov_b32_e32 v1, v33 +; GFX950-NEXT: v_mov_b32_e32 v2, v34 +; GFX950-NEXT: v_mov_b32_e32 v3, v35 +; GFX950-NEXT: s_waitcnt vmcnt(0) +; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_load_global_v64bf16: ; GFX10: ; %bb.0: diff --git a/llvm/test/CodeGen/AMDGPU/build_vector.ll b/llvm/test/CodeGen/AMDGPU/build_vector.ll index 7208eae..763f436 100644 --- a/llvm/test/CodeGen/AMDGPU/build_vector.ll +++ b/llvm/test/CodeGen/AMDGPU/build_vector.ll @@ -51,11 +51,11 @@ define amdgpu_kernel void @build_vector2 (ptr addrspace(1) %out) { ; GFX942-LABEL: build_vector2: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 -; GFX942-NEXT: v_mov_b32_e32 v2, 0 -; GFX942-NEXT: v_mov_b32_e32 v0, 5 -; GFX942-NEXT: v_mov_b32_e32 v1, 6 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 5 +; GFX942-NEXT: v_mov_b32_e32 v3, 6 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1] ; GFX942-NEXT: s_endpgm entry: store <2 x i32> <i32 5, i32 6>, ptr addrspace(1) %out @@ -116,13 +116,13 @@ define amdgpu_kernel void @build_vector4 (ptr addrspace(1) %out) { ; GFX942-LABEL: build_vector4: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 -; GFX942-NEXT: v_mov_b32_e32 v4, 0 -; GFX942-NEXT: v_mov_b32_e32 v0, 5 -; GFX942-NEXT: v_mov_b32_e32 v1, 6 -; GFX942-NEXT: v_mov_b32_e32 v2, 7 -; GFX942-NEXT: v_mov_b32_e32 v3, 8 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 5 +; GFX942-NEXT: v_mov_b32_e32 v3, 6 +; GFX942-NEXT: v_mov_b32_e32 v4, 7 +; GFX942-NEXT: v_mov_b32_e32 v5, 8 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] ; GFX942-NEXT: s_endpgm entry: store <4 x i32> <i32 5, i32 6, i32 7, i32 8>, ptr addrspace(1) %out @@ -307,13 +307,13 @@ define amdgpu_kernel void @build_v2i32_from_v4i16_shuffle(ptr addrspace(1) %out, ; GFX942-LABEL: build_v2i32_from_v4i16_shuffle: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-NEXT: s_lshl_b32 s3, s3, 16 ; GFX942-NEXT: s_lshl_b32 s2, s2, 16 -; GFX942-NEXT: v_mov_b32_e32 v0, s2 -; GFX942-NEXT: v_mov_b32_e32 v1, s3 -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX942-NEXT: v_mov_b32_e32 v2, s2 +; GFX942-NEXT: v_mov_b32_e32 v3, s3 +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1] ; GFX942-NEXT: s_endpgm entry: %shuf = shufflevector <4 x i16> %in, <4 x i16> zeroinitializer, <2 x i32> <i32 0, i32 2> diff --git a/llvm/test/CodeGen/AMDGPU/fix-crash-valu-hazard.ll b/llvm/test/CodeGen/AMDGPU/fix-crash-valu-hazard.ll index 8781196..4f752d1 100644 --- a/llvm/test/CodeGen/AMDGPU/fix-crash-valu-hazard.ll +++ b/llvm/test/CodeGen/AMDGPU/fix-crash-valu-hazard.ll @@ -8,10 +8,10 @@ define amdgpu_ps void @global_load_lds_dword_saddr(ptr addrspace(1) inreg nocapt ; GFX942-LABEL: global_load_lds_dword_saddr: ; GFX942: ; %bb.0: ; %main_body ; GFX942-NEXT: v_readfirstlane_b32 s2, v0 -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 ; GFX942-NEXT: s_mov_b32 m0, s2 ; GFX942-NEXT: s_nop 0 -; GFX942-NEXT: global_load_lds_dword v2, s[0:1] offset:32 nt +; GFX942-NEXT: global_load_lds_dword v1, s[0:1] offset:32 nt ; GFX942-NEXT: s_getpc_b64 s[0:1] ; GFX942-NEXT: s_add_u32 s0, s0, G@gotpcrel32@lo+4 ; GFX942-NEXT: s_addc_u32 s1, s1, G@gotpcrel32@hi+12 @@ -21,9 +21,9 @@ define amdgpu_ps void @global_load_lds_dword_saddr(ptr addrspace(1) inreg nocapt ; GFX942-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-NEXT: s_mul_i32 s3, s3, 10 ; GFX942-NEXT: s_mul_i32 s2, s2, 10 -; GFX942-NEXT: v_mov_b32_e32 v0, s2 -; GFX942-NEXT: v_mov_b32_e32 v1, s3 -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX942-NEXT: v_mov_b32_e32 v2, s2 +; GFX942-NEXT: v_mov_b32_e32 v3, s3 +; GFX942-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1] ; GFX942-NEXT: s_endpgm ; ; GFX90A-LABEL: global_load_lds_dword_saddr: diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll index 873fcee..6067194 100644 --- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll @@ -71,12 +71,12 @@ define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc: @@ -86,12 +86,12 @@ define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2) @@ -154,12 +154,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrsp ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc: @@ -169,12 +169,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrsp ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2) @@ -237,12 +237,12 @@ define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> % ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc: @@ -252,12 +252,12 @@ define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> % ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2) @@ -320,12 +320,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr add ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc: @@ -335,12 +335,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr add ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2) @@ -403,12 +403,12 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc: @@ -418,12 +418,12 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2) @@ -486,12 +486,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrsp ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc: @@ -501,12 +501,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrsp ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2) @@ -569,12 +569,12 @@ define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> % ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc: @@ -584,12 +584,12 @@ define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> % ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2) @@ -652,12 +652,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr add ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc: @@ -667,12 +667,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr add ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2) @@ -735,12 +735,12 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc: @@ -750,12 +750,12 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsr ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2) @@ -818,12 +818,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 4 offen glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc: @@ -833,12 +833,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 4 offen sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2) @@ -901,12 +901,12 @@ define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> % ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc: @@ -916,12 +916,12 @@ define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> % ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2) @@ -984,12 +984,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr add ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] -; GFX90A-NEXT: v_mov_b32_e32 v2, s10 -; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 glc slc -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: v_mov_b32_e32 v0, s10 +; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc: @@ -999,12 +999,12 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr add ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen offset:4 sc0 nt -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9] ; GFX942-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2) diff --git a/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll b/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll new file mode 100644 index 0000000..d1e82a0 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll @@ -0,0 +1,95 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GCN-SDAG %s +; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GCN-GISEL %s + +; Test that stores that may hit scratch are correctly promoted to SCOPE_SE. + +define void @test_scratch_store(ptr addrspace(5) %ptr, i32 %val) { +; GCN-LABEL: test_scratch_store: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: scratch_store_b32 v0, v1, off scope:SCOPE_SE +; GCN-NEXT: s_set_pc_i64 s[30:31] + store i32 %val, ptr addrspace(5) %ptr + ret void +} + +define void @test_unknown_flat_store(ptr %ptr, i32 %val) { +; GCN-LABEL: test_unknown_flat_store: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE +; GCN-NEXT: s_wait_dscnt 0x0 +; GCN-NEXT: s_set_pc_i64 s[30:31] + store i32 %val, ptr %ptr + ret void +} + +define void @test_flat_store_no_scratch_alloc(ptr %ptr, i32 %val) #0 { +; GCN-LABEL: test_flat_store_no_scratch_alloc: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: flat_store_b32 v[0:1], v2 +; GCN-NEXT: s_wait_dscnt 0x0 +; GCN-NEXT: s_set_pc_i64 s[30:31] + store i32 %val, ptr %ptr + ret void +} + +; TODO: handle +define void @test_flat_store_noalias_addrspace(ptr %ptr, i32 %val) { +; GCN-LABEL: test_flat_store_noalias_addrspace: +; GCN: ; %bb.0: +; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-NEXT: s_wait_kmcnt 0x0 +; GCN-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE +; GCN-NEXT: s_wait_dscnt 0x0 +; GCN-NEXT: s_set_pc_i64 s[30:31] + store i32 %val, ptr %ptr, !noalias.addrspace !{i32 5, i32 6} + ret void +} + +; TODO: would be nice to handle too +define void @test_flat_store_select(ptr addrspace(1) %a, ptr addrspace(3) %b, i1 %cond, i32 %val) { +; GCN-SDAG-LABEL: test_flat_store_select: +; GCN-SDAG: ; %bb.0: +; GCN-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-SDAG-NEXT: s_wait_kmcnt 0x0 +; GCN-SDAG-NEXT: v_cmp_ne_u32_e32 vcc_lo, -1, v2 +; GCN-SDAG-NEXT: v_and_b32_e32 v3, 1, v3 +; GCN-SDAG-NEXT: s_mov_b64 s[0:1], src_shared_base +; GCN-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc_lo +; GCN-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, s1, vcc_lo +; GCN-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GCN-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3 +; GCN-SDAG-NEXT: v_dual_cndmask_b32 v1, v5, v1 :: v_dual_cndmask_b32 v0, v2, v0 +; GCN-SDAG-NEXT: flat_store_b32 v[0:1], v4 scope:SCOPE_SE +; GCN-SDAG-NEXT: s_wait_dscnt 0x0 +; GCN-SDAG-NEXT: s_set_pc_i64 s[30:31] +; +; GCN-GISEL-LABEL: test_flat_store_select: +; GCN-GISEL: ; %bb.0: +; GCN-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GCN-GISEL-NEXT: s_wait_kmcnt 0x0 +; GCN-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, -1, v2 +; GCN-GISEL-NEXT: v_and_b32_e32 v3, 1, v3 +; GCN-GISEL-NEXT: s_mov_b64 s[0:1], src_shared_base +; GCN-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc_lo +; GCN-GISEL-NEXT: v_cndmask_b32_e64 v5, 0, s1, vcc_lo +; GCN-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GCN-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3 +; GCN-GISEL-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v5, v1 +; GCN-GISEL-NEXT: flat_store_b32 v[0:1], v4 scope:SCOPE_SE +; GCN-GISEL-NEXT: s_wait_dscnt 0x0 +; GCN-GISEL-NEXT: s_set_pc_i64 s[30:31] + %a.ascast = addrspacecast ptr addrspace(1) %a to ptr + %b.ascast = addrspacecast ptr addrspace(3) %b to ptr + %ptr = select i1 %cond, ptr %a.ascast, ptr %b.ascast + store i32 %val, ptr %ptr + ret void +} + +attributes #0 = { "amdgpu-no-flat-scratch-init" } diff --git a/llvm/test/CodeGen/AMDGPU/gfx90a-enc.ll b/llvm/test/CodeGen/AMDGPU/gfx90a-enc.ll index 99690e4..fe8edd5 100644 --- a/llvm/test/CodeGen/AMDGPU/gfx90a-enc.ll +++ b/llvm/test/CodeGen/AMDGPU/gfx90a-enc.ll @@ -1,4 +1,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -show-mc-encoding < %s | FileCheck -check-prefixes=GFX9,GFX908 %s + +; Make sure flag is ignored +; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -amdgpu-mfma-vgpr-form=1 -show-mc-encoding < %s | FileCheck -check-prefixes=GFX9,GFX908 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -show-mc-encoding < %s | FileCheck -check-prefixes=GFX9,GFX90A %s ; GFX9-DAG: buffer_load_format_xyzw v[{{[0-9:]+}}], v{{[0-9]+}}, s[{{[0-9:]+}}], 0 idxen ; encoding: diff --git a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll index fd644a3..3a898a9 100644 --- a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll +++ b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll @@ -124,27 +124,27 @@ define i32 @test_v64i32_load_store(ptr addrspace(1) %ptr, i32 %idx, ptr addrspac ; GCN-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 ; GCN-SDAG-NEXT: s_wait_kmcnt 0x0 ; GCN-SDAG-NEXT: s_clause 0xd -; GCN-SDAG-NEXT: scratch_store_b32 off, v40, s32 offset:52 -; GCN-SDAG-NEXT: scratch_store_b32 off, v41, s32 offset:48 -; GCN-SDAG-NEXT: scratch_store_b32 off, v42, s32 offset:44 -; GCN-SDAG-NEXT: scratch_store_b32 off, v43, s32 offset:40 -; GCN-SDAG-NEXT: scratch_store_b32 off, v44, s32 offset:36 -; GCN-SDAG-NEXT: scratch_store_b32 off, v45, s32 offset:32 -; GCN-SDAG-NEXT: scratch_store_b32 off, v56, s32 offset:28 -; GCN-SDAG-NEXT: scratch_store_b32 off, v57, s32 offset:24 -; GCN-SDAG-NEXT: scratch_store_b32 off, v58, s32 offset:20 -; GCN-SDAG-NEXT: scratch_store_b32 off, v59, s32 offset:16 -; GCN-SDAG-NEXT: scratch_store_b32 off, v60, s32 offset:12 -; GCN-SDAG-NEXT: scratch_store_b32 off, v61, s32 offset:8 -; GCN-SDAG-NEXT: scratch_store_b32 off, v62, s32 offset:4 -; GCN-SDAG-NEXT: scratch_store_b32 off, v63, s32 +; GCN-SDAG-NEXT: scratch_store_b32 off, v40, s32 offset:52 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v41, s32 offset:48 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v42, s32 offset:44 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v43, s32 offset:40 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v44, s32 offset:36 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v45, s32 offset:32 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v56, s32 offset:28 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v57, s32 offset:24 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v58, s32 offset:20 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v59, s32 offset:16 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v60, s32 offset:12 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v61, s32 offset:8 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v62, s32 offset:4 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v63, s32 scope:SCOPE_SE ; GCN-SDAG-NEXT: global_load_b128 v[6:9], v[0:1], off offset:224 ; GCN-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 ; GCN-SDAG-NEXT: s_wait_loadcnt 0x0 -; GCN-SDAG-NEXT: scratch_store_b128 off, v[6:9], s32 offset:56 ; 16-byte Folded Spill +; GCN-SDAG-NEXT: scratch_store_b128 off, v[6:9], s32 offset:56 scope:SCOPE_SE ; 16-byte Folded Spill ; GCN-SDAG-NEXT: global_load_b128 v[6:9], v[0:1], off offset:240 ; GCN-SDAG-NEXT: s_wait_loadcnt 0x0 -; GCN-SDAG-NEXT: scratch_store_b128 off, v[6:9], s32 offset:72 ; 16-byte Folded Spill +; GCN-SDAG-NEXT: scratch_store_b128 off, v[6:9], s32 offset:72 scope:SCOPE_SE ; 16-byte Folded Spill ; GCN-SDAG-NEXT: s_clause 0xd ; GCN-SDAG-NEXT: global_load_b128 v[10:13], v[0:1], off offset:192 ; GCN-SDAG-NEXT: global_load_b128 v[14:17], v[0:1], off offset:208 @@ -206,27 +206,27 @@ define i32 @test_v64i32_load_store(ptr addrspace(1) %ptr, i32 %idx, ptr addrspac ; GCN-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 ; GCN-GISEL-NEXT: s_wait_kmcnt 0x0 ; GCN-GISEL-NEXT: s_clause 0xf -; GCN-GISEL-NEXT: scratch_store_b32 off, v40, s32 offset:60 -; GCN-GISEL-NEXT: scratch_store_b32 off, v41, s32 offset:56 -; GCN-GISEL-NEXT: scratch_store_b32 off, v42, s32 offset:52 -; GCN-GISEL-NEXT: scratch_store_b32 off, v43, s32 offset:48 -; GCN-GISEL-NEXT: scratch_store_b32 off, v44, s32 offset:44 -; GCN-GISEL-NEXT: scratch_store_b32 off, v45, s32 offset:40 -; GCN-GISEL-NEXT: scratch_store_b32 off, v46, s32 offset:36 -; GCN-GISEL-NEXT: scratch_store_b32 off, v47, s32 offset:32 -; GCN-GISEL-NEXT: scratch_store_b32 off, v56, s32 offset:28 -; GCN-GISEL-NEXT: scratch_store_b32 off, v57, s32 offset:24 -; GCN-GISEL-NEXT: scratch_store_b32 off, v58, s32 offset:20 -; GCN-GISEL-NEXT: scratch_store_b32 off, v59, s32 offset:16 -; GCN-GISEL-NEXT: scratch_store_b32 off, v60, s32 offset:12 -; GCN-GISEL-NEXT: scratch_store_b32 off, v61, s32 offset:8 -; GCN-GISEL-NEXT: scratch_store_b32 off, v62, s32 offset:4 -; GCN-GISEL-NEXT: scratch_store_b32 off, v63, s32 +; GCN-GISEL-NEXT: scratch_store_b32 off, v40, s32 offset:60 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v41, s32 offset:56 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v42, s32 offset:52 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v43, s32 offset:48 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v44, s32 offset:44 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v45, s32 offset:40 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v46, s32 offset:36 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v47, s32 offset:32 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v56, s32 offset:28 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v57, s32 offset:24 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v58, s32 offset:20 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v59, s32 offset:16 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v60, s32 offset:12 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v61, s32 offset:8 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v62, s32 offset:4 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v63, s32 scope:SCOPE_SE ; GCN-GISEL-NEXT: s_wait_xcnt 0x8 ; GCN-GISEL-NEXT: v_dual_mov_b32 v46, v3 :: v_dual_mov_b32 v47, v4 ; GCN-GISEL-NEXT: global_load_b128 v[2:5], v[0:1], off offset:32 ; GCN-GISEL-NEXT: s_wait_loadcnt 0x0 -; GCN-GISEL-NEXT: scratch_store_b128 off, v[2:5], s32 offset:80 ; 16-byte Folded Spill +; GCN-GISEL-NEXT: scratch_store_b128 off, v[2:5], s32 offset:80 scope:SCOPE_SE ; 16-byte Folded Spill ; GCN-GISEL-NEXT: s_clause 0xe ; GCN-GISEL-NEXT: global_load_b128 v[6:9], v[0:1], off offset:48 ; GCN-GISEL-NEXT: global_load_b128 v[10:13], v[0:1], off offset:64 @@ -244,7 +244,7 @@ define i32 @test_v64i32_load_store(ptr addrspace(1) %ptr, i32 %idx, ptr addrspac ; GCN-GISEL-NEXT: global_load_b128 v[60:63], v[0:1], off offset:16 ; GCN-GISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:240 ; GCN-GISEL-NEXT: s_wait_loadcnt 0x0 -; GCN-GISEL-NEXT: scratch_store_b128 off, v[0:3], s32 offset:64 ; 16-byte Folded Spill +; GCN-GISEL-NEXT: scratch_store_b128 off, v[0:3], s32 offset:64 scope:SCOPE_SE ; 16-byte Folded Spill ; GCN-GISEL-NEXT: scratch_load_b128 v[0:3], off, s32 offset:80 th:TH_LOAD_LU ; 16-byte Folded Reload ; GCN-GISEL-NEXT: s_wait_loadcnt 0x0 ; GCN-GISEL-NEXT: s_clause 0xe @@ -299,10 +299,10 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt ; GCN-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 ; GCN-SDAG-NEXT: s_wait_kmcnt 0x0 ; GCN-SDAG-NEXT: s_clause 0x3 -; GCN-SDAG-NEXT: scratch_store_b32 off, v40, s32 offset:12 -; GCN-SDAG-NEXT: scratch_store_b32 off, v41, s32 offset:8 -; GCN-SDAG-NEXT: scratch_store_b32 off, v42, s32 offset:4 -; GCN-SDAG-NEXT: scratch_store_b32 off, v43, s32 +; GCN-SDAG-NEXT: scratch_store_b32 off, v40, s32 offset:12 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v41, s32 offset:8 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v42, s32 offset:4 scope:SCOPE_SE +; GCN-SDAG-NEXT: scratch_store_b32 off, v43, s32 scope:SCOPE_SE ; GCN-SDAG-NEXT: s_clause 0x7 ; GCN-SDAG-NEXT: global_load_b128 v[10:13], v[0:1], off offset:112 ; GCN-SDAG-NEXT: global_load_b128 v[18:21], v[0:1], off offset:96 @@ -385,12 +385,12 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt ; GCN-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 ; GCN-GISEL-NEXT: s_wait_kmcnt 0x0 ; GCN-GISEL-NEXT: s_clause 0x5 -; GCN-GISEL-NEXT: scratch_store_b32 off, v40, s32 offset:20 -; GCN-GISEL-NEXT: scratch_store_b32 off, v41, s32 offset:16 -; GCN-GISEL-NEXT: scratch_store_b32 off, v42, s32 offset:12 -; GCN-GISEL-NEXT: scratch_store_b32 off, v43, s32 offset:8 -; GCN-GISEL-NEXT: scratch_store_b32 off, v44, s32 offset:4 -; GCN-GISEL-NEXT: scratch_store_b32 off, v45, s32 +; GCN-GISEL-NEXT: scratch_store_b32 off, v40, s32 offset:20 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v41, s32 offset:16 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v42, s32 offset:12 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v43, s32 offset:8 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v44, s32 offset:4 scope:SCOPE_SE +; GCN-GISEL-NEXT: scratch_store_b32 off, v45, s32 scope:SCOPE_SE ; GCN-GISEL-NEXT: s_clause 0x7 ; GCN-GISEL-NEXT: global_load_b128 v[6:9], v[0:1], off offset:80 ; GCN-GISEL-NEXT: global_load_b128 v[10:13], v[0:1], off diff --git a/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll b/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll index 258aa9e..0a493e51 100644 --- a/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll +++ b/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll @@ -8,15 +8,15 @@ define protected amdgpu_kernel void @InferNothing(i32 %a, ptr %b, double %c) { ; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_ashr_i32 s7, s6, 31 -; CHECK-NEXT: v_mov_b32_e32 v0, s2 -; CHECK-NEXT: v_mov_b32_e32 v1, s3 +; CHECK-NEXT: v_mov_b32_e32 v2, s2 +; CHECK-NEXT: v_mov_b32_e32 v3, s3 ; CHECK-NEXT: s_lshl_b64 s[2:3], s[6:7], 3 ; CHECK-NEXT: s_add_u32 s0, s2, s0 ; CHECK-NEXT: s_addc_u32 s1, s3, s1 -; CHECK-NEXT: v_mov_b32_e32 v3, s1 -; CHECK-NEXT: v_add_co_u32_e64 v2, vcc, -8, s0 -; CHECK-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc -; CHECK-NEXT: flat_atomic_add_f64 v[2:3], v[0:1] +; CHECK-NEXT: v_mov_b32_e32 v1, s1 +; CHECK-NEXT: v_add_co_u32_e64 v0, vcc, -8, s0 +; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc +; CHECK-NEXT: flat_atomic_add_f64 v[0:1], v[2:3] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; CHECK-NEXT: buffer_wbinvl1_vol ; CHECK-NEXT: s_endpgm @@ -35,15 +35,15 @@ define protected amdgpu_kernel void @InferFadd(i32 %a, ptr addrspace(1) %b, doub ; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_ashr_i32 s7, s6, 31 -; CHECK-NEXT: v_mov_b32_e32 v0, s2 -; CHECK-NEXT: v_mov_b32_e32 v1, s3 +; CHECK-NEXT: v_mov_b32_e32 v2, s2 +; CHECK-NEXT: v_mov_b32_e32 v3, s3 ; CHECK-NEXT: s_lshl_b64 s[2:3], s[6:7], 3 ; CHECK-NEXT: s_add_u32 s0, s0, s2 ; CHECK-NEXT: s_addc_u32 s1, s1, s3 -; CHECK-NEXT: v_mov_b32_e32 v3, s1 -; CHECK-NEXT: v_add_co_u32_e64 v2, vcc, -8, s0 -; CHECK-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc -; CHECK-NEXT: flat_atomic_add_f64 v[2:3], v[0:1] +; CHECK-NEXT: v_mov_b32_e32 v1, s1 +; CHECK-NEXT: v_add_co_u32_e64 v0, vcc, -8, s0 +; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc +; CHECK-NEXT: flat_atomic_add_f64 v[0:1], v[2:3] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; CHECK-NEXT: buffer_wbinvl1_vol ; CHECK-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir index 11de6c8..06c3da0 100644 --- a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir +++ b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir @@ -32,32 +32,14 @@ # CHECK-NEXT: undef [[SPLIT0:%[0-9]+]].sub2_sub3:av_512_align2 = COPY undef $vgpr2_vgpr3 { # CHECK-NEXT: internal [[SPLIT0]].sub0:av_512_align2 = COPY undef $vgpr0 # CHECK-NEXT: } -# CHECK-NEXT: undef [[SPLIT1:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[SPLIT0]].sub2_sub3 { -# CHECK-NEXT: internal [[SPLIT1]].sub0:av_512_align2 = COPY [[SPLIT0]].sub0 -# CHECK-NEXT: } -# CHECK-NEXT: undef [[SPLIT2:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[SPLIT1]].sub2_sub3 { -# CHECK-NEXT: internal [[SPLIT2]].sub0:av_512_align2 = COPY [[SPLIT1]].sub0 -# CHECK-NEXT: } -# CHECK-NEXT: SI_SPILL_AV512_SAVE [[SPLIT2]], %stack.1, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.1, align 4, addrspace 5) -# CHECK-NEXT: [[RESTORE1:%[0-9]+]]:av_512_align2 = SI_SPILL_AV512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) -# CHECK-NEXT: undef [[SPLIT3:%[0-9]+]].sub0_sub1:av_512_align2 = COPY [[RESTORE1]].sub0_sub1 -# CHECK-NEXT: [[RESTORE2:%[0-9]+]]:av_512_align2 = SI_SPILL_AV512_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.1, align 4, addrspace 5) -# CHECK-NEXT: undef [[SPLIT3:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[RESTORE2]].sub2_sub3 { -# CHECK-NEXT: internal [[SPLIT3]].sub0:av_512_align2 = COPY [[RESTORE2]].sub0 -# CHECK-NEXT: } -# CHECK-NEXT: undef [[SPLIT4:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[SPLIT3]].sub2_sub3 { -# CHECK-NEXT: internal [[SPLIT4]].sub0:av_512_align2 = COPY [[SPLIT3]].sub0 -# CHECK-NEXT: } -# CHECK-NEXT: [[SPLIT5:%[0-9]+]].sub2:av_512_align2 = COPY [[SPLIT4]].sub3 -# CHECK-NEXT: undef [[SPLIT6:%[0-9]+]].sub0_sub1_sub2:av_512_align2 = COPY [[SPLIT5]].sub0_sub1_sub2 -# CHECK-NEXT: undef [[SPLIT7:%[0-9]+]].sub0_sub1_sub2:av_512_align2 = COPY [[SPLIT6]].sub0_sub1_sub2 -# CHECK-NEXT: undef [[SPLIT8:%[0-9]+]].sub0:av_512_align2 = COPY [[SPLIT4]].sub0 { -# CHECK-NEXT: internal [[SPLIT8]].sub2:av_512_align2 = COPY [[SPLIT4]].sub2 +# CHECK-NEXT: undef [[SPLIT2:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[SPLIT0]].sub2_sub3 { +# CHECK-NEXT: internal [[SPLIT2]].sub0:av_512_align2 = COPY [[SPLIT0]].sub0 # CHECK-NEXT: } -# CHECK-NEXT: [[SPLIT9:%[0-9]+]].sub3:av_512_align2 = COPY [[SPLIT8]].sub2 -# CHECK-NEXT: undef [[SPLIT10:%[0-9]+]].sub0_sub1_sub2_sub3:av_512_align2 = COPY [[SPLIT9]].sub0_sub1_sub2_sub3 -# CHECK-NEXT: undef [[SPLIT13:%[0-9]+]].sub0_sub1_sub2_sub3:vreg_512_align2 = COPY [[SPLIT10]].sub0_sub1_sub2_sub3 -# CHECK-NEXT: [[MFMA_USE1:%[0-9]+]].sub4:vreg_512_align2 = COPY [[SPLIT8]].sub0 +# CHECK-NEXT: [[RESTORE2:%[0-9]+]]:av_512_align2 = SI_SPILL_AV512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) +# CHECK-NEXT: [[MFMA_USE1:%[0-9]+]].sub0_sub1:vreg_512_align2 = COPY [[RESTORE2]].sub0_sub1 +# CHECK-NEXT: [[MFMA_USE1]].sub2:vreg_512_align2 = COPY [[SPLIT2]].sub3 +# CHECK-NEXT: [[MFMA_USE1]].sub3:vreg_512_align2 = COPY [[SPLIT2]].sub2 +# CHECK-NEXT: [[MFMA_USE1]].sub4:vreg_512_align2 = COPY [[SPLIT2]].sub0 # CHECK-NEXT: [[MFMA_USE1]].sub5:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec # CHECK-NEXT: [[MFMA_USE1]].sub6:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec # CHECK-NEXT: [[MFMA_USE1]].sub7:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll index 1ac75d3..d8c983a 100644 --- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll @@ -1331,16 +1331,16 @@ define amdgpu_kernel void @v_insertelement_v16bf16_3(ptr addrspace(1) %out, ptr ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX942-NEXT: s_load_dword s6, s[4:5], 0x10 ; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX942-NEXT: v_lshlrev_b32_e32 v8, 5, v0 -; GFX942-NEXT: v_mov_b32_e32 v9, 0x5040100 +; GFX942-NEXT: v_lshlrev_b32_e32 v4, 5, v0 +; GFX942-NEXT: v_mov_b32_e32 v5, 0x5040100 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_load_dwordx4 v[0:3], v8, s[2:3] -; GFX942-NEXT: global_load_dwordx4 v[4:7], v8, s[2:3] offset:16 +; GFX942-NEXT: global_load_dwordx4 v[0:3], v4, s[2:3] +; GFX942-NEXT: global_load_dwordx4 v[6:9], v4, s[2:3] offset:16 ; GFX942-NEXT: s_waitcnt vmcnt(1) -; GFX942-NEXT: v_perm_b32 v1, s6, v1, v9 +; GFX942-NEXT: v_perm_b32 v1, s6, v1, v5 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 -; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-NEXT: global_store_dwordx4 v4, v[6:9], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; GFX942-NEXT: s_endpgm %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 %tid.ext = sext i32 %tid to i64 diff --git a/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir new file mode 100644 index 0000000..58e9b0a --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir @@ -0,0 +1,94 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5 +# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -verify-regalloc -greedy-regclass-priority-trumps-globalness=1 -start-after=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck %s + +--- | + define void @temp_vgpr_to_agpr_should_not_undo_split_with_remat() #0 { + entry: + unreachable + } + + attributes #0 = { "amdgpu-agpr-alloc"="0,0" } +... + + +--- +name: temp_vgpr_to_agpr_should_not_undo_split_with_remat +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99' + stackPtrOffsetReg: '$sgpr32' + argumentInfo: + privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' } + kernargSegmentPtr: { reg: '$sgpr4_sgpr5' } + workGroupIDX: { reg: '$sgpr6' } + privateSegmentWaveByteOffset: { reg: '$sgpr7' } + workItemIDX: { reg: '$vgpr0' } + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + bb.0: + liveins: $vgpr0, $sgpr4_sgpr5 + ; CHECK-LABEL: name: temp_vgpr_to_agpr_should_not_undo_split_with_remat + ; CHECK: liveins: $vgpr0, $sgpr4_sgpr5 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $vgpr0 = IMPLICIT_DEF + ; CHECK-NEXT: dead renamable $vgpr1 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr1 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr2 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr3 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr4 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr5 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr6 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr7 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr8 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr9 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr10 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr11 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr12 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr13 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr14 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr15 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr16 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr17 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr18 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr19 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr20 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr21 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr22 = IMPLICIT_DEF + ; CHECK-NEXT: KILL killed renamable $vgpr2, killed renamable $vgpr3, killed renamable $vgpr4, killed renamable $vgpr5, killed renamable $vgpr6, killed renamable $vgpr7, killed renamable $vgpr8, killed renamable $vgpr9, killed renamable $vgpr10, killed renamable $vgpr11, killed renamable $vgpr12, killed renamable $vgpr13, killed renamable $vgpr14, killed renamable $vgpr15, killed renamable $vgpr16 + ; CHECK-NEXT: S_NOP 0, implicit-def renamable $vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38 + ; CHECK-NEXT: S_NOP 0, implicit-def renamable $vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54 + ; CHECK-NEXT: KILL killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr17, killed renamable $vgpr18, killed renamable $vgpr19, killed renamable $vgpr20, killed renamable $vgpr21, killed renamable $vgpr22 + ; CHECK-NEXT: S_NOP 0, implicit killed renamable $vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38, implicit killed renamable $vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54 + ; CHECK-NEXT: S_ENDPGM 0 + %1:vgpr_32 = IMPLICIT_DEF + %2:vgpr_32 = IMPLICIT_DEF + %2:vgpr_32 = IMPLICIT_DEF + %3:vgpr_32 = IMPLICIT_DEF + %4:vgpr_32 = IMPLICIT_DEF + %5:vgpr_32 = IMPLICIT_DEF + %6:vgpr_32 = IMPLICIT_DEF + %7:vgpr_32 = IMPLICIT_DEF + %8:vgpr_32 = IMPLICIT_DEF + %9:vgpr_32 = IMPLICIT_DEF + %10:vgpr_32 = IMPLICIT_DEF + %11:vgpr_32 = IMPLICIT_DEF + %12:vgpr_32 = IMPLICIT_DEF + %13:vgpr_32 = IMPLICIT_DEF + %14:vgpr_32 = IMPLICIT_DEF + %15:vgpr_32 = IMPLICIT_DEF + %16:vgpr_32 = IMPLICIT_DEF + %17:vgpr_32 = IMPLICIT_DEF + %18:vgpr_32 = IMPLICIT_DEF + %19:vgpr_32 = IMPLICIT_DEF + %20:vgpr_32 = IMPLICIT_DEF + %21:vgpr_32 = IMPLICIT_DEF + %22:vgpr_32 = IMPLICIT_DEF + %23:vgpr_32 = IMPLICIT_DEF + KILL %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17 + S_NOP 0, implicit-def %50:av_512 + S_NOP 0, implicit-def %51:av_512 + KILL %1, %2, %18, %19, %20, %21, %22, %23 + S_NOP 0, implicit %50, implicit %51 + S_ENDPGM 0 +... diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll index 85dd275..fcdad53 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll @@ -4,30 +4,30 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(<1 x i64> %L1) { ; GCN-LABEL: test_iglp_opt_rev_mfma_gemm: ; GCN: ; %bb.0: ; %entry -; GCN-NEXT: v_mov_b32_e32 v32, 0 -; GCN-NEXT: ds_read_b128 v[0:3], v32 +; GCN-NEXT: v_mov_b32_e32 v0, 0 +; GCN-NEXT: ds_read_b128 v[2:5], v0 ; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 -; GCN-NEXT: ds_read_b128 v[28:31], v32 offset:112 -; GCN-NEXT: ds_read_b128 v[24:27], v32 offset:96 -; GCN-NEXT: ds_read_b128 v[20:23], v32 offset:80 -; GCN-NEXT: ds_read_b128 v[16:19], v32 offset:64 -; GCN-NEXT: ds_read_b128 v[4:7], v32 offset:16 -; GCN-NEXT: ds_read_b128 v[8:11], v32 offset:32 -; GCN-NEXT: ds_read_b128 v[12:15], v32 offset:48 +; GCN-NEXT: ds_read_b128 v[30:33], v0 offset:112 +; GCN-NEXT: ds_read_b128 v[26:29], v0 offset:96 +; GCN-NEXT: ds_read_b128 v[22:25], v0 offset:80 +; GCN-NEXT: ds_read_b128 v[18:21], v0 offset:64 +; GCN-NEXT: ds_read_b128 v[6:9], v0 offset:16 +; GCN-NEXT: ds_read_b128 v[10:13], v0 offset:32 +; GCN-NEXT: ds_read_b128 v[14:17], v0 offset:48 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: ds_write_b128 v32, v[0:3] -; GCN-NEXT: v_mov_b32_e32 v0, 0 -; GCN-NEXT: v_mov_b32_e32 v1, v0 +; GCN-NEXT: ds_write_b128 v0, v[2:5] +; GCN-NEXT: v_mov_b32_e32 v2, 0 +; GCN-NEXT: v_mov_b32_e32 v3, v2 ; GCN-NEXT: s_cmp_lg_u64 s[0:1], 0 ; GCN-NEXT: ; iglp_opt mask(0x00000001) -; GCN-NEXT: ds_write_b128 v32, v[28:31] offset:112 -; GCN-NEXT: ds_write_b128 v32, v[24:27] offset:96 -; GCN-NEXT: ds_write_b128 v32, v[20:23] offset:80 -; GCN-NEXT: ds_write_b128 v32, v[16:19] offset:64 -; GCN-NEXT: ds_write_b128 v32, v[12:15] offset:48 -; GCN-NEXT: ds_write_b128 v32, v[8:11] offset:32 -; GCN-NEXT: ds_write_b128 v32, v[4:7] offset:16 -; GCN-NEXT: ds_write_b64 v32, v[0:1] +; GCN-NEXT: ds_write_b128 v0, v[30:33] offset:112 +; GCN-NEXT: ds_write_b128 v0, v[26:29] offset:96 +; GCN-NEXT: ds_write_b128 v0, v[22:25] offset:80 +; GCN-NEXT: ds_write_b128 v0, v[18:21] offset:64 +; GCN-NEXT: ds_write_b128 v0, v[14:17] offset:48 +; GCN-NEXT: ds_write_b128 v0, v[10:13] offset:32 +; GCN-NEXT: ds_write_b128 v0, v[6:9] offset:16 +; GCN-NEXT: ds_write_b64 v0, v[2:3] ; GCN-NEXT: s_endpgm entry: call void @llvm.amdgcn.iglp.opt(i32 1) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll index ed7d88b..dcac419 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll @@ -18,19 +18,22 @@ define amdgpu_ps <4 x float> @load_1d_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1 ; GCN-LABEL: load_1d_lwe: ; GCN: ; %bb.0: ; %main_body ; GCN-NEXT: v_mov_b32_e32 v8, 0 -; GCN-NEXT: v_mov_b32_e32 v6, v0 ; GCN-NEXT: v_mov_b32_e32 v9, v8 ; GCN-NEXT: v_mov_b32_e32 v10, v8 ; GCN-NEXT: v_mov_b32_e32 v11, v8 ; GCN-NEXT: v_mov_b32_e32 v12, v8 -; GCN-NEXT: v_mov_b32_e32 v0, v8 -; GCN-NEXT: v_mov_b32_e32 v1, v9 -; GCN-NEXT: v_mov_b32_e32 v2, v10 -; GCN-NEXT: v_mov_b32_e32 v3, v11 -; GCN-NEXT: v_mov_b32_e32 v4, v12 -; GCN-NEXT: image_load v[0:4], v6, s[0:7] dmask:0xf unorm lwe +; GCN-NEXT: v_mov_b32_e32 v2, v8 +; GCN-NEXT: v_mov_b32_e32 v3, v9 +; GCN-NEXT: v_mov_b32_e32 v4, v10 +; GCN-NEXT: v_mov_b32_e32 v5, v11 +; GCN-NEXT: v_mov_b32_e32 v6, v12 +; GCN-NEXT: image_load v[2:6], v0, s[0:7] dmask:0xf unorm lwe ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dword v8, v4, s[8:9] +; GCN-NEXT: v_mov_b32_e32 v0, v2 +; GCN-NEXT: v_mov_b32_e32 v1, v3 +; GCN-NEXT: v_mov_b32_e32 v2, v4 +; GCN-NEXT: v_mov_b32_e32 v3, v5 +; GCN-NEXT: global_store_dword v8, v6, s[8:9] ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: ; return to shader part epilog main_body: @@ -75,6 +78,27 @@ main_body: } define amdgpu_ps <4 x float> @load_cube_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice) { +; GCN-LABEL: load_cube_lwe: +; GCN: ; %bb.0: ; %main_body +; GCN-NEXT: v_mov_b32_e32 v10, 0 +; GCN-NEXT: v_mov_b32_e32 v11, v10 +; GCN-NEXT: v_mov_b32_e32 v12, v10 +; GCN-NEXT: v_mov_b32_e32 v13, v10 +; GCN-NEXT: v_mov_b32_e32 v14, v10 +; GCN-NEXT: v_mov_b32_e32 v4, v10 +; GCN-NEXT: v_mov_b32_e32 v5, v11 +; GCN-NEXT: v_mov_b32_e32 v6, v12 +; GCN-NEXT: v_mov_b32_e32 v7, v13 +; GCN-NEXT: v_mov_b32_e32 v8, v14 +; GCN-NEXT: image_load v[4:8], v[0:2], s[0:7] dmask:0xf unorm lwe da +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, v4 +; GCN-NEXT: v_mov_b32_e32 v1, v5 +; GCN-NEXT: v_mov_b32_e32 v2, v6 +; GCN-NEXT: v_mov_b32_e32 v3, v7 +; GCN-NEXT: global_store_dword v10, v8, s[8:9] +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: ; return to shader part epilog main_body: %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.cube.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0) %v.vec = extractvalue {<4 x float>, i32} %v, 0 @@ -106,6 +130,27 @@ main_body: } define amdgpu_ps <4 x float> @load_2darray_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice) { +; GCN-LABEL: load_2darray_lwe: +; GCN: ; %bb.0: ; %main_body +; GCN-NEXT: v_mov_b32_e32 v10, 0 +; GCN-NEXT: v_mov_b32_e32 v11, v10 +; GCN-NEXT: v_mov_b32_e32 v12, v10 +; GCN-NEXT: v_mov_b32_e32 v13, v10 +; GCN-NEXT: v_mov_b32_e32 v14, v10 +; GCN-NEXT: v_mov_b32_e32 v4, v10 +; GCN-NEXT: v_mov_b32_e32 v5, v11 +; GCN-NEXT: v_mov_b32_e32 v6, v12 +; GCN-NEXT: v_mov_b32_e32 v7, v13 +; GCN-NEXT: v_mov_b32_e32 v8, v14 +; GCN-NEXT: image_load v[4:8], v[0:2], s[0:7] dmask:0xf unorm lwe da +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, v4 +; GCN-NEXT: v_mov_b32_e32 v1, v5 +; GCN-NEXT: v_mov_b32_e32 v2, v6 +; GCN-NEXT: v_mov_b32_e32 v3, v7 +; GCN-NEXT: global_store_dword v10, v8, s[8:9] +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: ; return to shader part epilog main_body: %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2darray.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0) %v.vec = extractvalue {<4 x float>, i32} %v, 0 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll index 1585a2c..303ea50 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll @@ -1,6 +1,7 @@ -; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX908 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX908 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x2bf16(<2 x i16>, <2 x i16>, <32 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x2bf16(<2 x i16>, <2 x i16>, <16 x float>, i32, i32, i32) @@ -9,50 +10,199 @@ declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16(<2 x i16>, <2 x i16>, <16 declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x8bf16(<2 x i16>, <2 x i16>, <4 x float>, i32, i32, i32) declare i32 @llvm.amdgcn.workitem.id.x() -; GCN-LABEL: {{^}}test_mfma_f32_32x32x2bf16: -; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; GCN-DAG: s_load_dwordx16 -; GCN-DAG: s_load_dwordx16 -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX908-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX90A-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GCN: v_mfma_f32_32x32x2bf16 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX908-COUNT-32: v_accvgpr_read_b32 -; GFX908: global_store_dwordx4 -; GFX90A-NOT: v_accvgpr_read_b32 -; GFX90A-COUNT-8: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}], define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 { +; GFX908-LABEL: test_mfma_f32_32x32x2bf16: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX908-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v0, s16 +; GFX908-NEXT: v_mov_b32_e32 v1, s17 +; GFX908-NEXT: v_mov_b32_e32 v2, s18 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s21 +; GFX908-NEXT: v_mov_b32_e32 v1, s22 +; GFX908-NEXT: v_mov_b32_e32 v2, s23 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s24 +; GFX908-NEXT: v_mov_b32_e32 v1, s25 +; GFX908-NEXT: v_mov_b32_e32 v2, s26 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s27 +; GFX908-NEXT: v_mov_b32_e32 v1, s28 +; GFX908-NEXT: v_mov_b32_e32 v2, s29 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s30 +; GFX908-NEXT: v_mov_b32_e32 v1, s31 +; GFX908-NEXT: v_mov_b32_e32 v2, s0 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s1 +; GFX908-NEXT: v_mov_b32_e32 v1, s2 +; GFX908-NEXT: v_mov_b32_e32 v2, s3 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s4 +; GFX908-NEXT: v_mov_b32_e32 v1, s5 +; GFX908-NEXT: v_mov_b32_e32 v2, s6 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s7 +; GFX908-NEXT: v_mov_b32_e32 v1, s8 +; GFX908-NEXT: v_mov_b32_e32 v2, s9 +; GFX908-NEXT: v_mov_b32_e32 v3, s19 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s10 +; GFX908-NEXT: v_mov_b32_e32 v1, s11 +; GFX908-NEXT: v_mov_b32_e32 v2, s12 +; GFX908-NEXT: v_mov_b32_e32 v5, s20 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s13 +; GFX908-NEXT: v_mov_b32_e32 v1, s14 +; GFX908-NEXT: v_mov_b32_e32 v2, s15 +; GFX908-NEXT: v_mov_b32_e32 v3, 1 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a31, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, 2 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a28 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a16 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a20 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a8 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a12 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; GFX908-NEXT: s_endpgm +; +; GFX90A-LABEL: test_mfma_f32_32x32x2bf16: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v1, 1 +; GFX90A-NEXT: v_mov_b32_e32 v2, 2 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s16 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s17 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s18 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s19 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s20 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s21 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s22 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s23 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, s24 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, s25 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, s26 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, s27 +; GFX90A-NEXT: v_accvgpr_write_b32 a12, s28 +; GFX90A-NEXT: v_accvgpr_write_b32 a13, s29 +; GFX90A-NEXT: v_accvgpr_write_b32 a14, s30 +; GFX90A-NEXT: v_accvgpr_write_b32 a15, s31 +; GFX90A-NEXT: v_accvgpr_write_b32 a16, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a17, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a18, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a19, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a20, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a21, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a22, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a23, s7 +; GFX90A-NEXT: v_accvgpr_write_b32 a24, s8 +; GFX90A-NEXT: v_accvgpr_write_b32 a25, s9 +; GFX90A-NEXT: v_accvgpr_write_b32 a26, s10 +; GFX90A-NEXT: v_accvgpr_write_b32 a27, s11 +; GFX90A-NEXT: v_accvgpr_write_b32 a28, s12 +; GFX90A-NEXT: v_accvgpr_write_b32 a29, s13 +; GFX90A-NEXT: v_accvgpr_write_b32 a30, s14 +; GFX90A-NEXT: v_accvgpr_write_b32 a31, s15 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v1, v2, a[0:31] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: global_store_dwordx4 v0, a[24:27], s[34:35] offset:96 +; GFX90A-NEXT: global_store_dwordx4 v0, a[28:31], s[34:35] offset:112 +; GFX90A-NEXT: global_store_dwordx4 v0, a[16:19], s[34:35] offset:64 +; GFX90A-NEXT: global_store_dwordx4 v0, a[20:23], s[34:35] offset:80 +; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[34:35] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[34:35] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[34:35] +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[34:35] offset:16 +; GFX90A-NEXT: s_endpgm bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg %a = bitcast i32 1 to <2 x i16> @@ -62,18 +212,109 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x2bf16: -; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; GCN-DAG: s_load_dwordx16 -; GFX908-DAG-COUNT-16: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX90A-COUNT-16: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GCN: v_mfma_f32_16x16x2bf16 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX908-COUNT-16: v_accvgpr_read_b32 -; GFX908: global_store_dwordx4 -; GFX90A-NOT: v_accvgpr_read_b32 -; GFX90A-COUNT-4: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}], define amdgpu_kernel void @test_mfma_f32_16x16x2bf16(ptr addrspace(1) %arg) #0 { +; GFX908-LABEL: test_mfma_f32_16x16x2bf16: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-NEXT: v_mov_b32_e32 v12, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v13, s0 +; GFX908-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v13 +; GFX908-NEXT: v_mov_b32_e32 v13, s3 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v13 +; GFX908-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-NEXT: v_mov_b32_e32 v13, s6 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v13 +; GFX908-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-NEXT: v_mov_b32_e32 v13, s9 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v13 +; GFX908-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-NEXT: v_mov_b32_e32 v13, s12 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v13 +; GFX908-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-NEXT: v_mov_b32_e32 v13, s15 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v13 +; GFX908-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a4 +; GFX908-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48 +; GFX908-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:32 +; GFX908-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:16 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] +; GFX908-NEXT: s_endpgm +; +; GFX90A-LABEL: test_mfma_f32_16x16x2bf16: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v0, 1 +; GFX90A-NEXT: v_mov_b32_e32 v1, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX90A-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %a = bitcast i32 1 to <2 x i16> @@ -83,18 +324,53 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_4x4x2bf16: -; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; GCN: s_load_dwordx4 -; GFX908-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX90A-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GCN: v_mfma_f32_4x4x2bf16 [[RES:a\[[0-9]+:[0-9]+\]]], [[ONE]], [[TWO]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX908-COUNT-4: v_accvgpr_read_b32 -; GFX908: global_store_dwordx4 -; GFX90A-NOT: v_accvgpr_read_b32 -; GFX90A: global_store_dwordx4 v{{[0-9]+}}, [[RES]], define amdgpu_kernel void @test_mfma_f32_4x4x2bf16(ptr addrspace(1) %arg) #0 { +; GFX908-LABEL: test_mfma_f32_4x4x2bf16: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v5, s0 +; GFX908-NEXT: v_mov_b32_e32 v2, s1 +; GFX908-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-NEXT: v_mov_b32_e32 v5, s3 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_mfma_f32_4x4x2bf16 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX908-NEXT: s_nop 3 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX908-NEXT: s_endpgm +; +; GFX90A-LABEL: test_mfma_f32_4x4x2bf16: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v0, 1 +; GFX90A-NEXT: v_mov_b32_e32 v2, 2 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_4x4x2bf16 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 4 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] +; GFX90A-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %a = bitcast i32 1 to <2 x i16> @@ -104,18 +380,110 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x4bf16: -; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; GCN-DAG: s_load_dwordx16 -; GFX908-DAG-COUNT-16: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX90A-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GCN: v_mfma_f32_32x32x4bf16 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX908-COUNT-16: v_accvgpr_read_b32 -; GFX908: global_store_dwordx4 -; GFX90A-NOT: v_accvgpr_read_b32 -; GFX90A-COUNT-4: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}], define amdgpu_kernel void @test_mfma_f32_32x32x4bf16(ptr addrspace(1) %arg) #0 { +; GFX908-LABEL: test_mfma_f32_32x32x4bf16: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a4 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a0 +; GFX908-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48 +; GFX908-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32 +; GFX908-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16 +; GFX908-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] +; GFX908-NEXT: s_endpgm +; +; GFX90A-LABEL: test_mfma_f32_32x32x4bf16: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v0, 1 +; GFX90A-NEXT: v_mov_b32_e32 v1, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX90A-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %a = bitcast i32 1 to <2 x i16> @@ -125,18 +493,55 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x8bf16: -; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; GCN: s_load_dwordx4 -; GFX908-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX90A-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GCN: v_mfma_f32_16x16x8bf16 [[RES:a\[[0-9]+:[0-9]+\]]], [[ONE]], [[TWO]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX908-COUNT-4: v_accvgpr_read_b32 -; GFX908: global_store_dwordx4 -; GFX90A-NOT: v_accvgpr_read_b32 -; GFX90A: global_store_dwordx4 v{{[0-9]+}}, [[RES]], define amdgpu_kernel void @test_mfma_f32_16x16x8bf16(ptr addrspace(1) %arg) #0 { +; GFX908-LABEL: test_mfma_f32_16x16x8bf16: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v5, s0 +; GFX908-NEXT: v_mov_b32_e32 v2, s1 +; GFX908-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-NEXT: v_mov_b32_e32 v5, s3 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_mfma_f32_16x16x8bf16 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX908-NEXT: s_endpgm +; +; GFX90A-LABEL: test_mfma_f32_16x16x8bf16: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v0, 1 +; GFX90A-NEXT: v_mov_b32_e32 v2, 2 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_16x16x8bf16 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] +; GFX90A-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %a = bitcast i32 1 to <2 x i16> @@ -147,3 +552,5 @@ bb: } attributes #0 = { "amdgpu-flat-work-group-size"="1,256" } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GCN: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll index 4c26961..ff77d5cc 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll @@ -1,5 +1,8 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=VGPR,GFX90A-VGPR %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=VGPR,GFX942-VGPR %s declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16.1k(<4 x i16>, <4 x i16>, <32 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x4bf16.1k(<4 x i16>, <4 x i16>, <16 x float>, i32, i32, i32) @@ -10,17 +13,238 @@ declare <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double, double, <4 x doubl declare double @llvm.amdgcn.mfma.f64.4x4x4f64(double, double, double, i32, i32, i32) declare i32 @llvm.amdgcn.workitem.id.x() -; GCN-LABEL: {{^}}test_mfma_f32_32x32x4bf16_1k: -; GCN-DAG: s_load_dwordx16 -; GCN-DAG: s_load_dwordx16 -; GCN-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GCN-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX90A: v_mfma_f32_32x32x4bf16_1k a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9]+}}], v[[[TWO]]:{{[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f32_32x32x4_2b_bf16 a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9+]}}], v[[[TWO]]:{{[0-9+]}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN-COUNT-8: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x4bf16_1k(ptr addrspace(1) %arg) #0 { +; GFX90A-LABEL: test_mfma_f32_32x32x4bf16_1k: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_mov_b32_e32 v2, 1 +; GFX90A-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-NEXT: v_mov_b32_e32 v0, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s16 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s17 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s18 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s19 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s20 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s21 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s22 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s23 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, s24 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, s25 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, s26 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, s27 +; GFX90A-NEXT: v_accvgpr_write_b32 a12, s28 +; GFX90A-NEXT: v_accvgpr_write_b32 a13, s29 +; GFX90A-NEXT: v_accvgpr_write_b32 a14, s30 +; GFX90A-NEXT: v_accvgpr_write_b32 a15, s31 +; GFX90A-NEXT: v_accvgpr_write_b32 a16, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a17, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a18, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a19, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a20, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a21, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a22, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a23, s7 +; GFX90A-NEXT: v_accvgpr_write_b32 a24, s8 +; GFX90A-NEXT: v_accvgpr_write_b32 a25, s9 +; GFX90A-NEXT: v_accvgpr_write_b32 a26, s10 +; GFX90A-NEXT: v_accvgpr_write_b32 a27, s11 +; GFX90A-NEXT: v_accvgpr_write_b32 a28, s12 +; GFX90A-NEXT: v_accvgpr_write_b32 a29, s13 +; GFX90A-NEXT: v_accvgpr_write_b32 a30, s14 +; GFX90A-NEXT: v_accvgpr_write_b32 a31, s15 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_32x32x4bf16_1k a[0:31], v[2:3], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: global_store_dwordx4 v1, a[24:27], s[34:35] offset:96 +; GFX90A-NEXT: global_store_dwordx4 v1, a[28:31], s[34:35] offset:112 +; GFX90A-NEXT: global_store_dwordx4 v1, a[16:19], s[34:35] offset:64 +; GFX90A-NEXT: global_store_dwordx4 v1, a[20:23], s[34:35] offset:80 +; GFX90A-NEXT: global_store_dwordx4 v1, a[8:11], s[34:35] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v1, a[12:15], s[34:35] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[34:35] +; GFX90A-NEXT: global_store_dwordx4 v1, a[4:7], s[34:35] offset:16 +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f32_32x32x4bf16_1k: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 1 +; GFX942-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX942-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s16 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s17 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s18 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s19 +; GFX942-NEXT: v_accvgpr_write_b32 a4, s20 +; GFX942-NEXT: v_accvgpr_write_b32 a5, s21 +; GFX942-NEXT: v_accvgpr_write_b32 a6, s22 +; GFX942-NEXT: v_accvgpr_write_b32 a7, s23 +; GFX942-NEXT: v_accvgpr_write_b32 a8, s24 +; GFX942-NEXT: v_accvgpr_write_b32 a9, s25 +; GFX942-NEXT: v_accvgpr_write_b32 a10, s26 +; GFX942-NEXT: v_accvgpr_write_b32 a11, s27 +; GFX942-NEXT: v_accvgpr_write_b32 a12, s28 +; GFX942-NEXT: v_accvgpr_write_b32 a13, s29 +; GFX942-NEXT: v_accvgpr_write_b32 a14, s30 +; GFX942-NEXT: v_accvgpr_write_b32 a15, s31 +; GFX942-NEXT: v_accvgpr_write_b32 a16, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a17, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a18, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a19, s3 +; GFX942-NEXT: v_accvgpr_write_b32 a20, s4 +; GFX942-NEXT: v_accvgpr_write_b32 a21, s5 +; GFX942-NEXT: v_accvgpr_write_b32 a22, s6 +; GFX942-NEXT: v_accvgpr_write_b32 a23, s7 +; GFX942-NEXT: v_accvgpr_write_b32 a24, s8 +; GFX942-NEXT: v_accvgpr_write_b32 a25, s9 +; GFX942-NEXT: v_accvgpr_write_b32 a26, s10 +; GFX942-NEXT: v_accvgpr_write_b32 a27, s11 +; GFX942-NEXT: v_accvgpr_write_b32 a28, s12 +; GFX942-NEXT: v_accvgpr_write_b32 a29, s13 +; GFX942-NEXT: v_accvgpr_write_b32 a30, s14 +; GFX942-NEXT: v_accvgpr_write_b32 a31, s15 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f32_32x32x4_2b_bf16 a[0:31], v[2:3], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 2 +; GFX942-NEXT: global_store_dwordx4 v1, a[24:27], s[34:35] offset:96 +; GFX942-NEXT: global_store_dwordx4 v1, a[28:31], s[34:35] offset:112 +; GFX942-NEXT: global_store_dwordx4 v1, a[16:19], s[34:35] offset:64 +; GFX942-NEXT: global_store_dwordx4 v1, a[20:23], s[34:35] offset:80 +; GFX942-NEXT: global_store_dwordx4 v1, a[8:11], s[34:35] offset:32 +; GFX942-NEXT: global_store_dwordx4 v1, a[12:15], s[34:35] offset:48 +; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[34:35] +; GFX942-NEXT: global_store_dwordx4 v1, a[4:7], s[34:35] offset:16 +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x4bf16_1k: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v33, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v34, 1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v35, v33 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v32, 2 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, s16 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, s17 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, s18 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, s19 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, s20 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, s21 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, s22 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, s23 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, s24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v9, s25 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v10, s26 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v11, s27 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v12, s28 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v13, s29 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v14, s30 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v15, s31 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, s0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, s1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v18, s2 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v19, s3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v20, s4 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v21, s5 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v22, s6 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v23, s7 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v24, s8 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v25, s9 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v26, s10 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v27, s11 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v28, s12 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v29, s13 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v30, s14 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v31, s15 +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f32_32x32x4bf16_1k v[0:31], v[34:35], v[32:33], v[0:31] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 2 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[24:27], s[34:35] offset:96 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[28:31], s[34:35] offset:112 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[16:19], s[34:35] offset:64 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[20:23], s[34:35] offset:80 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[8:11], s[34:35] offset:32 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[12:15], s[34:35] offset:48 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[0:3], s[34:35] +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16 +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x4bf16_1k: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v33, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v34, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v35, v33 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v32, 2 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, s16 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, s17 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, s18 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, s19 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, s20 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, s21 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, s22 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, s23 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, s24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, s25 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, s26 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, s27 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, s28 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, s29 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v14, s30 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v15, s31 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, s0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, s1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, s2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, s3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v20, s4 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v21, s5 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v22, s6 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v23, s7 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v24, s8 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v25, s9 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v26, s10 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v27, s11 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v28, s12 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v29, s13 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v30, s14 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v31, s15 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x4_2b_bf16 v[0:31], v[34:35], v[32:33], v[0:31] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[24:27], s[34:35] offset:96 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[28:31], s[34:35] offset:112 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[16:19], s[34:35] offset:64 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[20:23], s[34:35] offset:80 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[8:11], s[34:35] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[12:15], s[34:35] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[0:3], s[34:35] +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16 +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg %a = bitcast i64 1 to <4 x i16> @@ -30,16 +254,134 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x4bf16_1k: -; GCN-DAG: s_load_dwordx16 -; GCN-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GCN-COUNT-16: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX90A: v_mfma_f32_16x16x4bf16_1k a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9]+}}], v[[[TWO]]:{{[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f32_16x16x4_4b_bf16 a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9+]}}], v[[[TWO]]:{{[0-9+]}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN-COUNT-4: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_16x16x4bf16_1k(ptr addrspace(1) %arg) #0 { +; GFX90A-LABEL: test_mfma_f32_16x16x4bf16_1k: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_mov_b32_e32 v2, 1 +; GFX90A-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-NEXT: v_mov_b32_e32 v0, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_16x16x4bf16_1k a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: global_store_dwordx4 v1, a[12:15], s[16:17] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v1, a[8:11], s[16:17] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v1, a[4:7], s[16:17] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[16:17] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f32_16x16x4bf16_1k: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 1 +; GFX942-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f32_16x16x4_4b_bf16 a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 2 +; GFX942-NEXT: global_store_dwordx4 v1, a[12:15], s[16:17] offset:48 +; GFX942-NEXT: global_store_dwordx4 v1, a[8:11], s[16:17] offset:32 +; GFX942-NEXT: global_store_dwordx4 v1, a[4:7], s[16:17] offset:16 +; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[16:17] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x4bf16_1k: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v18, 1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v19, v17 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 2 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f32_16x16x4bf16_1k v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 2 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[12:15], s[16:17] offset:48 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[8:11], s[16:17] offset:32 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4bf16_1k: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, v17 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 2 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[12:15], s[16:17] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[8:11], s[16:17] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %a = bitcast i64 1 to <4 x i16> @@ -49,16 +391,82 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_4x4x4bf16_1k: -; GCN-DAG: s_load_dwordx4 -; GCN-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX90A: v_mfma_f32_4x4x4bf16_1k [[RES:a\[[0-9]+:[0-9]+\]]], v[[[ONE]]:{{[0-9]+}}], v[[[TWO]]:{{[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f32_4x4x4_16b_bf16 [[RES:a\[[0-9]+:[0-9]+\]]], v[[[ONE]]:{{[0-9+]}}], v[[[TWO]]:{{[0-9+]}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[RES]], define amdgpu_kernel void @test_mfma_f32_4x4x4bf16_1k(ptr addrspace(1) %arg) #0 { +; GFX90A-LABEL: test_mfma_f32_4x4x4bf16_1k: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_mov_b32_e32 v2, 1 +; GFX90A-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-NEXT: v_mov_b32_e32 v0, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_4x4x4bf16_1k a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 4 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f32_4x4x4bf16_1k: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 1 +; GFX942-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f32_4x4x4_16b_bf16 a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_nop 4 +; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f32_4x4x4bf16_1k: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, 1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v5 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 2 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f32_4x4x4bf16_1k v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 4 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_4x4x4bf16_1k: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v5 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 2 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_4x4x4_16b_bf16 v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 4 +; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %a = bitcast i64 1 to <4 x i16> @@ -68,16 +476,136 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x8bf16_1k: -; GCN-DAG: s_load_dwordx16 -; GCN-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GCN-COUNT-16: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX90A: v_mfma_f32_32x32x8bf16_1k a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9]+}}], v[[[TWO]]:{{[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f32_32x32x8_bf16 a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9+]}}], v[[[TWO]]:{{[0-9+]}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN-COUNT-4: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x8bf16_1k(ptr addrspace(1) %arg) #0 { +; GFX90A-LABEL: test_mfma_f32_32x32x8bf16_1k: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_mov_b32_e32 v2, 1 +; GFX90A-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-NEXT: v_mov_b32_e32 v0, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_32x32x8bf16_1k a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: global_store_dwordx4 v1, a[12:15], s[16:17] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v1, a[8:11], s[16:17] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v1, a[4:7], s[16:17] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[16:17] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f32_32x32x8bf16_1k: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 1 +; GFX942-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f32_32x32x8_bf16 a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 2 +; GFX942-NEXT: global_store_dwordx4 v1, a[12:15], s[16:17] offset:48 +; GFX942-NEXT: global_store_dwordx4 v1, a[8:11], s[16:17] offset:32 +; GFX942-NEXT: global_store_dwordx4 v1, a[4:7], s[16:17] offset:16 +; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[16:17] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x8bf16_1k: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v18, 1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v19, v17 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 2 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f32_32x32x8bf16_1k v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 2 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[12:15], s[16:17] offset:48 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[8:11], s[16:17] offset:32 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8bf16_1k: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, v17 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 2 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x8_bf16 v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[12:15], s[16:17] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[8:11], s[16:17] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %a = bitcast i64 1 to <4 x i16> @@ -87,16 +615,84 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x16bf16_1k: -; GCN-DAG: s_load_dwordx4 -; GCN-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX90A: v_mfma_f32_16x16x16bf16_1k [[RES:a\[[0-9]+:[0-9]+\]]], v[[[ONE]]:{{[0-9]+}}], v[[[TWO]]:{{[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f32_16x16x16_bf16 [[RES:a\[[0-9]+:[0-9]+\]]], v[[[ONE]]:{{[0-9+]}}], v[[[TWO]]:{{[0-9+]}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[RES]], define amdgpu_kernel void @test_mfma_f32_16x16x16bf16_1k(ptr addrspace(1) %arg) #0 { +; GFX90A-LABEL: test_mfma_f32_16x16x16bf16_1k: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_mov_b32_e32 v2, 1 +; GFX90A-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-NEXT: v_mov_b32_e32 v0, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_16x16x16bf16_1k a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f32_16x16x16bf16_1k: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 1 +; GFX942-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f32_16x16x16_bf16 a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_nop 6 +; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x16bf16_1k: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, 1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v5 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 2 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f32_16x16x16bf16_1k v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 2 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_16x16x16bf16_1k: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v5 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 2 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x16_bf16 v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 6 +; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %a = bitcast i64 1 to <4 x i16> @@ -106,13 +702,70 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_4x4x4f64: -; GFX90A: v_mfma_f64_4x4x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 0{{$}} -; GFX90A: v_mfma_f64_4x4x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_4x4x4_4b_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 0{{$}} -; GFX942: v_mfma_f64_4x4x4_4b_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx2 define amdgpu_kernel void @test_mfma_f64_4x4x4f64(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_4x4x4f64: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_4x4x4f64 a[0:1], v[0:1], v[2:3], 0 +; GFX90A-NEXT: s_nop 3 +; GFX90A-NEXT: v_mfma_f64_4x4x4f64 a[0:1], v[0:1], v[2:3], a[0:1] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: global_store_dwordx2 v0, a[0:1], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_4x4x4f64: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], v[2:3], 0 +; GFX942-NEXT: s_nop 3 +; GFX942-NEXT: v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], v[2:3], a[0:1] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: global_store_dwordx2 v0, a[0:1], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_4x4x4f64: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_4x4x4f64 v[4:5], v[0:1], v[2:3], 0 +; GFX90A-VGPR-NEXT: s_nop 3 +; GFX90A-VGPR-NEXT: v_mfma_f64_4x4x4f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_4x4x4f64: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_4x4x4_4b_f64 v[4:5], v[0:1], v[2:3], 0 +; GFX942-VGPR-NEXT: s_nop 3 +; GFX942-VGPR-NEXT: v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %a, double %b, double 0.0, i32 0, i32 0, i32 0) %mai.2 = tail call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %a, double %b, double %mai.1, i32 1, i32 2, i32 3) @@ -120,13 +773,110 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64: -; GCN: s_load_dwordx8 -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_mov_b32_e32 v2, s10 +; GFX90A-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v3, s11 +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[12:13], s[12:13] op_sel:[0,1] +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[2:3], v[0:1], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[8:9] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0 +; GFX942-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[2:3], v[0:1], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[8:9] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v10, s10 +; GFX90A-VGPR-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v11, s11 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[12:13], s[12:13] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[10:11], v[8:9], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, s10 +; GFX942-VGPR-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, s11 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[12:13] +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[10:11], v[8:9], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x double>, ptr addrspace(1) %arg %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %in.1, i32 1, i32 2, i32 3) @@ -134,14 +884,78 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_0: -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 0{{$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 0{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_0(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], 0 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], 0 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 0 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 0 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> zeroinitializer, i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -149,14 +963,78 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_int_neg1: -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], -1{{$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], -1{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_neg1(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], -1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], -1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 -1 to double)), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -164,14 +1042,78 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_1: -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 1.0{{$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 1.0{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], 1.0 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], 1.0 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 1.0 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 1.0 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double 1.0), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -179,14 +1121,78 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_neg1: -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], -1.0{{$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], -1.0{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], -1.0 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], -1.0 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1.0 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1.0 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double -1.0), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -194,14 +1200,78 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_int_64: -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 64{{$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 64{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], 64 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], 64 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 64 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 64 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 64 to double)), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -209,23 +1279,116 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits: -; GCN: v_accvgpr_write_b32 a[[A_LOW_BITS_0:[0-9]+]], 0{{$}} -; GCN: v_accvgpr_write_b32 a[[A_HIGH_BITS_0:[0-9]+]], 64 -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_HIGH_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_HIGH_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a[[LAST_CONST_REG:[0-9]+]], a[[A_HIGH_BITS_0]] - -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, 64 +; GFX90A-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a1 +; GFX90A-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a5, a1 +; GFX90A-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a7, a1 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, 64 +; GFX942-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_accvgpr_mov_b32 a3, a1 +; GFX942-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a5, a1 +; GFX942-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a7, a1 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, 64 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, v1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v1 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[12:13], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[2:9], v[10:11], v[12:13], v[2:9] +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[2:9], v[10:11], v[12:13], v[2:9] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 64 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v1 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], v[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], v[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], v[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], v[0:1] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[2:9], v[10:11], v[12:13], v[2:9] +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[2:9], v[10:11], v[12:13], v[2:9] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 274877906944 to double)), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -233,23 +1396,110 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low: -; GCN: v_accvgpr_write_b32 a[[A_LOW_BITS_0:[0-9]+]], 64{{$}} -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a[[LAST_CONST_REG:[0-9]+]], a[[A_LOW_BITS_0]] - -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, 64 +; GFX90A-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a7, a0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: v_accvgpr_write_b32 a0, 64 +; GFX942-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a7, a0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, 64 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v0 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 64 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v0 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 274877907008 to double)), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -257,23 +1507,110 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low: -; GCN: v_accvgpr_write_b32 a[[A_LOW_BITS_0:[0-9]+]], 1.0 -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a[[LAST_CONST_REG:[0-9]+]], a[[A_LOW_BITS_0]] - -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a7, a0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX942-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a7, a0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v0 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v0 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (<2 x float> splat (float 1.0) to double)), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -281,26 +1618,236 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_imm: -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}]{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}]{{$}} -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_imm: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0x3ff00000 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, v0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_mov_b32_e32 v2, s2 +; GFX90A-NEXT: v_mov_b32_e32 v3, s3 +; GFX90A-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[2:3], v[0:1], a[0:7] +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_imm: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX942-NEXT: v_mov_b32_e32 v0, 0x3ff00000 +; GFX942-NEXT: v_accvgpr_write_b32 a7, v0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b32_e32 v2, s2 +; GFX942-NEXT: v_mov_b32_e32 v3, s3 +; GFX942-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[2:3], v[0:1], a[0:7] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_imm: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, 0x3ff00000 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v12, s2 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v13, s3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[2:9], v[12:13], v[10:11], v[2:9] +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_imm: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, 0x3ff00000 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, s2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, s3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], v[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], v[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], v[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], v[0:1] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[2:9], v[12:13], v[10:11], v[2:9] +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> <double 0.0, double 0.0, double 0.0, double 1.0>, i32 0, i32 0, i32 0) store <4 x double> %mai.1, ptr addrspace(1) %arg ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_lit: -; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}} -; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x405ec000 -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}]{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}]{{$}} -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_lit: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0x405ec000 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, v0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_mov_b32_e32 v2, s2 +; GFX90A-NEXT: v_mov_b32_e32 v3, s3 +; GFX90A-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a1 +; GFX90A-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a5, a1 +; GFX90A-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a7, a1 +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[2:3], v[0:1], a[0:7] +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_lit: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: v_mov_b32_e32 v0, 0x405ec000 +; GFX942-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, v0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b32_e32 v2, s2 +; GFX942-NEXT: v_mov_b32_e32 v3, s3 +; GFX942-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a3, a1 +; GFX942-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a5, a1 +; GFX942-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a7, a1 +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[2:3], v[0:1], a[0:7] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_lit: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, 0x405ec000 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v12, s2 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v13, s3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, v1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v1 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[2:9], v[12:13], v[10:11], v[2:9] +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_lit: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 0x405ec000 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, s2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, s3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v1 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], v[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], v[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], v[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], v[0:1] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[2:9], v[12:13], v[10:11], v[2:9] +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> <double 123.0, double 123.0, double 123.0, double 123.0>, i32 0, i32 0, i32 0) store <4 x double> %mai.1, ptr addrspace(1) %arg @@ -308,3 +1855,6 @@ bb: } attributes #0 = { "amdgpu-flat-work-group-size"="1,256" } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GCN: {{.*}} +; VGPR: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll index b792a12..7d85d34 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll @@ -1,12 +1,13 @@ -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,VGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -global-isel < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL,VGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -stress-regalloc=10 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,AGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -stress-regalloc=10 -global-isel < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL,AGPRCD %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-SDAG,GFX942-VGPRCD-SDAG %s +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-GISEL,GFX942-VGPRCD-GISEL %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 -stress-regalloc=10 < %s | FileCheck --check-prefixes=GFX942,GFX942-AGPRCD,GFX942-SDAG,GFX942-AGPRCD-SDAG %s +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 -stress-regalloc=10 < %s | FileCheck --check-prefixes=GFX942,GFX942-AGPRCD,GFX942-GISEL,GFX942-AGPRCD-GISEL %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,VGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL,VGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -stress-regalloc=10 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,AGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -stress-regalloc=10 -global-isel < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL,AGPRCD %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950,GFX950-VGPRCD,GFX950-SDAG,GFX950-VGPRCD-SDAG %s +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950,GFX950-VGPRCD,GFX950-GISEL,GFX950-VGPRCD-GISEL %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -stress-regalloc=10 < %s | FileCheck --check-prefixes=GFX950,GFX950-AGPRCD,GFX950-SDAG,GFX950-AGPRCD-SDAG %s +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx950 -stress-regalloc=10 < %s | FileCheck --check-prefixes=GFX950,GFX950-AGPRCD,GFX950-GISEL,GFX950-AGPRCD-GISEL %s declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x32.i8(i64, i64, <4 x i32>, i32, i32, i32) declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x16.i8(i64, i64, <16 x i32>, i32, i32, i32) @@ -33,17 +34,132 @@ declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.fp8(<2 x i32>, <4 x i3 declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.bf8(<2 x i32>, <4 x i32>, <16 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.fp8(<2 x i32>, <4 x i32>, <16 x float>, i32, i32, i32) -; GCN-LABEL: {{^}}test_mfma_i32_16x16x32i8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_i32_16x16x32_i8 a[{{[0-9]+:[0-9]+}}], v[[[TWO]]:[[ONE]]], v[[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_i32_16x16x32_i8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_i32_16x16x32i8(ptr addrspace(1) %arg) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_i32_16x16x32i8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 5 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_i32_16x16x32i8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 6 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x32.i8(i64 4294967298, i64 12884901892, <4 x i32> %in.1, i32 1, i32 2, i32 3) @@ -51,17 +167,154 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_i32_32x32x16i8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-16: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_i32_32x32x16_i8 a[{{[0-9]+:[0-9]+}}], v[[[TWO]]:[[ONE]]], v[[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_i32_32x32x16_i8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN-COUNT-4: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_i32_32x32x16i8(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_i32_32x32x16i8: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-NEXT: s_nop 7 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_i32_32x32x16i8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 7 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX950-SDAG-LABEL: test_mfma_i32_32x32x16i8: +; GFX950-SDAG: ; %bb.0: ; %bb +; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-SDAG-NEXT: s_nop 1 +; GFX950-SDAG-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-SDAG-NEXT: s_nop 7 +; GFX950-SDAG-NEXT: s_nop 2 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_i32_32x32x16i8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 7 +; GFX950-GISEL-NEXT: s_nop 2 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x16.i8(i64 4294967298, i64 12884901892, <16 x i32> %in.1, i32 1, i32 2, i32 3) @@ -69,17 +322,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x32_bf8_bf8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_16x16x32_bf8_bf8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_16x16x32_bf8_bf8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_16x16x32_bf8_bf8(ptr addrspace(1) %arg) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 5 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 6 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf8.bf8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) @@ -87,17 +455,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x32_bf8_fp8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_16x16x32_bf8_fp8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_16x16x32_bf8_fp8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_16x16x32_bf8_fp8(ptr addrspace(1) %arg) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 5 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 6 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf8.fp8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) @@ -105,17 +588,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x32_fp8_bf8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_16x16x32_fp8_bf8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_16x16x32_fp8_bf8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_16x16x32_fp8_bf8(ptr addrspace(1) %arg) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 5 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 6 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.bf8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) @@ -123,17 +721,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x32_fp8_fp8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_16x16x32_fp8_fp8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_16x16x32_fp8_fp8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_16x16x32_fp8_fp8(ptr addrspace(1) %arg) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 5 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 6 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) @@ -141,17 +854,154 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x16_bf8_bf8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_32x32x16_bf8_bf8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_32x32x16_bf8_bf8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x16_bf8_bf8(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_bf8: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-NEXT: s_nop 7 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_bf8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 7 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_bf8: +; GFX950-SDAG: ; %bb.0: ; %bb +; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-SDAG-NEXT: s_nop 1 +; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-SDAG-NEXT: s_nop 7 +; GFX950-SDAG-NEXT: s_nop 2 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_bf8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 7 +; GFX950-GISEL-NEXT: s_nop 2 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf8.bf8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) @@ -159,17 +1009,154 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x16_bf8_fp8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_32x32x16_bf8_fp8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_32x32x16_bf8_fp8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x16_bf8_fp8(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_fp8: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-NEXT: s_nop 7 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_fp8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 7 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_fp8: +; GFX950-SDAG: ; %bb.0: ; %bb +; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-SDAG-NEXT: s_nop 1 +; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-SDAG-NEXT: s_nop 7 +; GFX950-SDAG-NEXT: s_nop 2 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_fp8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 7 +; GFX950-GISEL-NEXT: s_nop 2 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf8.fp8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) @@ -177,17 +1164,154 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x16_fp8_bf8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_32x32x16_fp8_bf8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_32x32x16_fp8_bf8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x16_fp8_bf8(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_bf8: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-NEXT: s_nop 7 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_bf8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 7 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_bf8: +; GFX950-SDAG: ; %bb.0: ; %bb +; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-SDAG-NEXT: s_nop 1 +; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-SDAG-NEXT: s_nop 7 +; GFX950-SDAG-NEXT: s_nop 2 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_bf8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 7 +; GFX950-GISEL-NEXT: s_nop 2 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.fp8.bf8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) @@ -195,17 +1319,154 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x16_fp8_fp8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_32x32x16_fp8_fp8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_32x32x16_fp8_fp8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x16_fp8_fp8(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_fp8: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-NEXT: s_nop 7 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_fp8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 7 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_fp8: +; GFX950-SDAG: ; %bb.0: ; %bb +; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-SDAG-NEXT: s_nop 1 +; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-SDAG-NEXT: s_nop 7 +; GFX950-SDAG-NEXT: s_nop 2 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_fp8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 7 +; GFX950-GISEL-NEXT: s_nop 2 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.fp8.fp8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) @@ -213,15 +1474,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_f32_16x16x32_f16: -; GCN: s_load_dwordx4 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x32_f16 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_f32_16x16x32_f16(ptr addrspace(1) %arg, <4 x half> %a, <8 x half> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX942-AGPRCD: ; %bb.0: ; %bb +; GFX942-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-NEXT: s_nop 1 +; GFX942-AGPRCD-NEXT: v_smfmac_f32_16x16x32_f16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-NEXT: s_nop 5 +; GFX942-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX942-AGPRCD-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX950-AGPRCD: ; %bb.0: ; %bb +; GFX950-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-NEXT: s_nop 1 +; GFX950-AGPRCD-NEXT: v_smfmac_f32_16x16x32_f16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-NEXT: s_nop 6 +; GFX950-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX950-AGPRCD-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.f16(<4 x half> %a, <8 x half> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -229,18 +1607,278 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_f32_32x32x16_f16: -; GCN: s_load_dwordx16 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x16_f16 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_f32_32x32x16_f16(ptr addrspace(1) %arg, <4 x half> %a, <8 x half> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x16.f16(<4 x half> %a, <8 x half> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -248,15 +1886,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_f32_16x16x32_bf16: -; GCN: s_load_dwordx4 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x32_bf16 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_f32_16x16x32_bf16(ptr addrspace(1) %arg, <4 x i16> %a, <8 x i16> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX942-AGPRCD: ; %bb.0: ; %bb +; GFX942-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-NEXT: s_nop 1 +; GFX942-AGPRCD-NEXT: v_smfmac_f32_16x16x32_bf16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-NEXT: s_nop 5 +; GFX942-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX942-AGPRCD-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX950-AGPRCD: ; %bb.0: ; %bb +; GFX950-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-NEXT: s_nop 1 +; GFX950-AGPRCD-NEXT: v_smfmac_f32_16x16x32_bf16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-NEXT: s_nop 6 +; GFX950-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX950-AGPRCD-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.bf16(<4 x i16> %a, <8 x i16> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -264,18 +2019,278 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_f32_32x32x16_bf16: -; GCN: s_load_dwordx16 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x16_bf16 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_f32_32x32x16_bf16(ptr addrspace(1) %arg, <4 x i16> %a, <8 x i16> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x16.bf16(<4 x i16> %a, <8 x i16> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -283,15 +2298,214 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_16x16x64_i8: -; GCN: s_load_dwordx4 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_i32_16x16x64_i8 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_i32_16x16x64_i8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <4 x i32> @llvm.amdgcn.smfmac.i32.16x16x64.i8(<2 x i32> %a, <4 x i32> %b, <4 x i32> %in.1, i32 %idx, i32 1, i32 2) @@ -299,18 +2513,310 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_32x32x32_i8: -; GCN: s_load_dwordx16 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_i32_32x32x32_i8 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_i32_32x32x32_i8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x32.i8(<2 x i32> %a, <4 x i32> %b, <16 x i32> %in.1, i32 %idx, i32 1, i32 2) @@ -318,15 +2824,214 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_16x16x64_bf8_bf8: -; GCN: s_load_dwordx4 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x64_bf8_bf8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf8.bf8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -334,15 +3039,214 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_16x16x64_bf8_fp8: -; GCN: s_load_dwordx4 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x64_bf8_fp8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf8.fp8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -350,15 +3254,214 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_16x16x64_fp8_bf8: -; GCN: s_load_dwordx4 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x64_fp8_bf8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.fp8.bf8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -366,15 +3469,214 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_16x16x64_fp8_fp8: -; GCN: s_load_dwordx4 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x64_fp8_fp8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.fp8.fp8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -382,18 +3684,310 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_32x32x32_bf8_bf8: -; GCN: s_load_dwordx16 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x32_bf8_bf8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.bf8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -401,18 +3995,310 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_32x32x32_bf8_fp8: -; GCN: s_load_dwordx16 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x32_bf8_fp8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.fp8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -420,18 +4306,310 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_32x32x32_fp8_bf8: -; GCN: s_load_dwordx16 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x32_fp8_bf8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.bf8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -439,18 +4617,310 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_32x32x32_fp8_fp8: -; GCN: s_load_dwordx16 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x32_fp8_fp8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.fp8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -459,3 +4929,8 @@ bb: } attributes #0 = { "amdgpu-flat-work-group-size"="1,256" } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GFX942: {{.*}} +; GFX942-VGPRCD: {{.*}} +; GFX950: {{.*}} +; GFX950-VGPRCD: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll index 452033f..d358837 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll @@ -15,9 +15,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GCN-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; GCN-NEXT: v_mov_b64_e32 v[12:13], 48 -; GCN-NEXT: v_mov_b64_e32 v[14:15], 32 -; GCN-NEXT: v_mov_b64_e32 v[16:17], 16 +; GCN-NEXT: v_mov_b64_e32 v[8:9], 48 +; GCN-NEXT: v_mov_b64_e32 v[10:11], 32 +; GCN-NEXT: v_mov_b64_e32 v[12:13], 16 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; GCN-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -39,42 +39,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x ; GCN-NEXT: v_accvgpr_write_b32 a13, s21 ; GCN-NEXT: v_accvgpr_write_b32 a14, s22 ; GCN-NEXT: v_accvgpr_write_b32 a15, s23 -; GCN-NEXT: v_mov_b64_e32 v[18:19], 0 -; GCN-NEXT: v_mov_b32_e32 v8, s16 +; GCN-NEXT: v_mov_b64_e32 v[14:15], 0 +; GCN-NEXT: v_mov_b32_e32 v16, s16 ; GCN-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[0:3], v[4:7], a[0:15] ; GCN-NEXT: v_mov_b32_e32 v0, s20 ; GCN-NEXT: v_mov_b32_e32 v1, s21 ; GCN-NEXT: v_mov_b32_e32 v2, s22 ; GCN-NEXT: v_mov_b32_e32 v3, s23 -; GCN-NEXT: v_mov_b32_e32 v9, s17 -; GCN-NEXT: v_mov_b32_e32 v10, s18 -; GCN-NEXT: v_mov_b32_e32 v11, s19 +; GCN-NEXT: v_mov_b32_e32 v17, s17 +; GCN-NEXT: v_mov_b32_e32 v18, s18 +; GCN-NEXT: v_mov_b32_e32 v19, s19 ; GCN-NEXT: s_nop 4 -; GCN-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[10:11], v[16:19], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_nop 0 ; GCN-NEXT: v_mov_b32_e32 v0, s8 ; GCN-NEXT: v_mov_b32_e32 v1, s9 ; GCN-NEXT: v_mov_b32_e32 v2, s10 ; GCN-NEXT: v_mov_b32_e32 v3, s11 -; GCN-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_nop 0 ; GCN-NEXT: v_mov_b32_e32 v0, s12 ; GCN-NEXT: v_mov_b32_e32 v1, s13 ; GCN-NEXT: v_mov_b32_e32 v2, s14 ; GCN-NEXT: v_mov_b32_e32 v3, s15 -; GCN-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_endpgm %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0) @@ -88,9 +88,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0 ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GCN-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; GCN-NEXT: v_mov_b64_e32 v[12:13], 48 -; GCN-NEXT: v_mov_b64_e32 v[14:15], 32 -; GCN-NEXT: v_mov_b64_e32 v[16:17], 16 +; GCN-NEXT: v_mov_b64_e32 v[8:9], 48 +; GCN-NEXT: v_mov_b64_e32 v[10:11], 32 +; GCN-NEXT: v_mov_b64_e32 v[12:13], 16 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; GCN-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -112,42 +112,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0 ; GCN-NEXT: v_accvgpr_write_b32 a13, s21 ; GCN-NEXT: v_accvgpr_write_b32 a14, s22 ; GCN-NEXT: v_accvgpr_write_b32 a15, s23 -; GCN-NEXT: v_mov_b64_e32 v[18:19], 0 -; GCN-NEXT: v_mov_b32_e32 v8, s16 +; GCN-NEXT: v_mov_b64_e32 v[14:15], 0 +; GCN-NEXT: v_mov_b32_e32 v16, s16 ; GCN-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1 ; GCN-NEXT: v_mov_b32_e32 v0, s20 ; GCN-NEXT: v_mov_b32_e32 v1, s21 ; GCN-NEXT: v_mov_b32_e32 v2, s22 ; GCN-NEXT: v_mov_b32_e32 v3, s23 -; GCN-NEXT: v_mov_b32_e32 v9, s17 -; GCN-NEXT: v_mov_b32_e32 v10, s18 -; GCN-NEXT: v_mov_b32_e32 v11, s19 +; GCN-NEXT: v_mov_b32_e32 v17, s17 +; GCN-NEXT: v_mov_b32_e32 v18, s18 +; GCN-NEXT: v_mov_b32_e32 v19, s19 ; GCN-NEXT: s_nop 4 -; GCN-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[10:11], v[16:19], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_nop 0 ; GCN-NEXT: v_mov_b32_e32 v0, s8 ; GCN-NEXT: v_mov_b32_e32 v1, s9 ; GCN-NEXT: v_mov_b32_e32 v2, s10 ; GCN-NEXT: v_mov_b32_e32 v3, s11 -; GCN-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_nop 0 ; GCN-NEXT: v_mov_b32_e32 v0, s12 ; GCN-NEXT: v_mov_b32_e32 v1, s13 ; GCN-NEXT: v_mov_b32_e32 v2, s14 ; GCN-NEXT: v_mov_b32_e32 v3, s15 -; GCN-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_endpgm %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 2, i32 3, i32 1) @@ -252,7 +252,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg ; GCN-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GCN-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 ; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; GCN-NEXT: v_mov_b32_e32 v12, 0 +; GCN-NEXT: v_mov_b32_e32 v8, 0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; GCN-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -274,40 +274,40 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg ; GCN-NEXT: v_accvgpr_write_b32 a18, s10 ; GCN-NEXT: v_accvgpr_write_b32 a17, s9 ; GCN-NEXT: v_accvgpr_write_b32 a16, s8 -; GCN-NEXT: v_mov_b32_e32 v8, s20 -; GCN-NEXT: v_mov_b32_e32 v9, s21 +; GCN-NEXT: v_mov_b32_e32 v10, s20 +; GCN-NEXT: v_mov_b32_e32 v11, s21 ; GCN-NEXT: v_mfma_f32_32x32x16_bf16 a[0:15], v[0:3], v[4:7], a[16:31] -; GCN-NEXT: v_mov_b32_e32 v10, s22 -; GCN-NEXT: v_mov_b32_e32 v11, s23 +; GCN-NEXT: v_mov_b32_e32 v12, s22 +; GCN-NEXT: v_mov_b32_e32 v13, s23 ; GCN-NEXT: v_mov_b32_e32 v0, s16 ; GCN-NEXT: v_mov_b32_e32 v1, s17 ; GCN-NEXT: v_mov_b32_e32 v2, s18 ; GCN-NEXT: v_mov_b32_e32 v3, s19 -; GCN-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, v[10:13], s[0:1] offset:48 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_nop 0 ; GCN-NEXT: v_mov_b32_e32 v0, s12 ; GCN-NEXT: v_mov_b32_e32 v1, s13 ; GCN-NEXT: v_mov_b32_e32 v2, s14 ; GCN-NEXT: v_mov_b32_e32 v3, s15 -; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_nop 0 ; GCN-NEXT: v_mov_b32_e32 v0, s8 ; GCN-NEXT: v_mov_b32_e32 v1, s9 ; GCN-NEXT: v_mov_b32_e32 v2, s10 ; GCN-NEXT: v_mov_b32_e32 v3, s11 -; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_endpgm %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0) @@ -322,7 +322,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa ; GCN-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GCN-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 ; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; GCN-NEXT: v_mov_b32_e32 v12, 0 +; GCN-NEXT: v_mov_b32_e32 v8, 0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; GCN-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -344,40 +344,40 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa ; GCN-NEXT: v_accvgpr_write_b32 a18, s10 ; GCN-NEXT: v_accvgpr_write_b32 a17, s9 ; GCN-NEXT: v_accvgpr_write_b32 a16, s8 -; GCN-NEXT: v_mov_b32_e32 v8, s20 -; GCN-NEXT: v_mov_b32_e32 v9, s21 +; GCN-NEXT: v_mov_b32_e32 v10, s20 +; GCN-NEXT: v_mov_b32_e32 v11, s21 ; GCN-NEXT: v_mfma_f32_32x32x16_bf16 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:3 -; GCN-NEXT: v_mov_b32_e32 v10, s22 -; GCN-NEXT: v_mov_b32_e32 v11, s23 +; GCN-NEXT: v_mov_b32_e32 v12, s22 +; GCN-NEXT: v_mov_b32_e32 v13, s23 ; GCN-NEXT: v_mov_b32_e32 v0, s16 ; GCN-NEXT: v_mov_b32_e32 v1, s17 ; GCN-NEXT: v_mov_b32_e32 v2, s18 ; GCN-NEXT: v_mov_b32_e32 v3, s19 -; GCN-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, v[10:13], s[0:1] offset:48 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_nop 0 ; GCN-NEXT: v_mov_b32_e32 v0, s12 ; GCN-NEXT: v_mov_b32_e32 v1, s13 ; GCN-NEXT: v_mov_b32_e32 v2, s14 ; GCN-NEXT: v_mov_b32_e32 v3, s15 -; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_nop 0 ; GCN-NEXT: v_mov_b32_e32 v0, s8 ; GCN-NEXT: v_mov_b32_e32 v1, s9 ; GCN-NEXT: v_mov_b32_e32 v2, s10 ; GCN-NEXT: v_mov_b32_e32 v3, s11 -; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1 +; GCN-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_endpgm %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 1, i32 2, i32 3) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll index 9bdae28f..21465be 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll @@ -394,9 +394,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal ; SDAG: ; %bb.0: ; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; SDAG-NEXT: v_mov_b64_e32 v[12:13], 48 -; SDAG-NEXT: v_mov_b64_e32 v[14:15], 32 -; SDAG-NEXT: v_mov_b64_e32 v[16:17], 16 +; SDAG-NEXT: v_mov_b64_e32 v[8:9], 48 +; SDAG-NEXT: v_mov_b64_e32 v[10:11], 32 +; SDAG-NEXT: v_mov_b64_e32 v[12:13], 16 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -418,42 +418,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal ; SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; SDAG-NEXT: v_accvgpr_write_b32 a15, s23 -; SDAG-NEXT: v_mov_b64_e32 v[18:19], 0 -; SDAG-NEXT: v_mov_b32_e32 v8, s16 +; SDAG-NEXT: v_mov_b64_e32 v[14:15], 0 +; SDAG-NEXT: v_mov_b32_e32 v16, s16 ; SDAG-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] ; SDAG-NEXT: v_mov_b32_e32 v0, s20 ; SDAG-NEXT: v_mov_b32_e32 v1, s21 ; SDAG-NEXT: v_mov_b32_e32 v2, s22 ; SDAG-NEXT: v_mov_b32_e32 v3, s23 -; SDAG-NEXT: v_mov_b32_e32 v9, s17 -; SDAG-NEXT: v_mov_b32_e32 v10, s18 -; SDAG-NEXT: v_mov_b32_e32 v11, s19 +; SDAG-NEXT: v_mov_b32_e32 v17, s17 +; SDAG-NEXT: v_mov_b32_e32 v18, s18 +; SDAG-NEXT: v_mov_b32_e32 v19, s19 ; SDAG-NEXT: s_nop 4 -; SDAG-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[10:11], v[16:19], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s8 ; SDAG-NEXT: v_mov_b32_e32 v1, s9 ; SDAG-NEXT: v_mov_b32_e32 v2, s10 ; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s12 ; SDAG-NEXT: v_mov_b32_e32 v1, s13 ; SDAG-NEXT: v_mov_b32_e32 v2, s14 ; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; @@ -518,9 +518,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal ; HEURRC: ; %bb.0: ; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 48 -; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 32 -; HEURRC-NEXT: v_mov_b64_e32 v[16:17], 16 +; HEURRC-NEXT: v_mov_b64_e32 v[8:9], 48 +; HEURRC-NEXT: v_mov_b64_e32 v[10:11], 32 +; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 16 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) ; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -542,42 +542,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal ; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 ; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 ; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 -; HEURRC-NEXT: v_mov_b64_e32 v[18:19], 0 -; HEURRC-NEXT: v_mov_b32_e32 v8, s16 +; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 0 +; HEURRC-NEXT: v_mov_b32_e32 v16, s16 ; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] ; HEURRC-NEXT: v_mov_b32_e32 v0, s20 ; HEURRC-NEXT: v_mov_b32_e32 v1, s21 ; HEURRC-NEXT: v_mov_b32_e32 v2, s22 ; HEURRC-NEXT: v_mov_b32_e32 v3, s23 -; HEURRC-NEXT: v_mov_b32_e32 v9, s17 -; HEURRC-NEXT: v_mov_b32_e32 v10, s18 -; HEURRC-NEXT: v_mov_b32_e32 v11, s19 +; HEURRC-NEXT: v_mov_b32_e32 v17, s17 +; HEURRC-NEXT: v_mov_b32_e32 v18, s18 +; HEURRC-NEXT: v_mov_b32_e32 v19, s19 ; HEURRC-NEXT: s_nop 4 -; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[10:11], v[16:19], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s8 ; HEURRC-NEXT: v_mov_b32_e32 v1, s9 ; HEURRC-NEXT: v_mov_b32_e32 v2, s10 ; HEURRC-NEXT: v_mov_b32_e32 v3, s11 -; HEURRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s12 ; HEURRC-NEXT: v_mov_b32_e32 v1, s13 ; HEURRC-NEXT: v_mov_b32_e32 v2, s14 ; HEURRC-NEXT: v_mov_b32_e32 v3, s15 -; HEURRC-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_endpgm ; @@ -585,9 +585,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal ; VGPRRC: ; %bb.0: ; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 48 -; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 32 -; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], 16 +; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], 48 +; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], 32 +; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 16 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) ; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27] ; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25] @@ -601,43 +601,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal ; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] ; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] ; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] -; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], 0 -; VGPRRC-NEXT: v_mov_b32_e32 v40, s16 +; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 0 +; VGPRRC-NEXT: v_mov_b32_e32 v48, s16 ; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] -; VGPRRC-NEXT: v_mov_b32_e32 v41, s17 -; VGPRRC-NEXT: v_mov_b32_e32 v42, s18 -; VGPRRC-NEXT: v_mov_b32_e32 v43, s19 +; VGPRRC-NEXT: v_mov_b32_e32 v49, s17 +; VGPRRC-NEXT: v_mov_b32_e32 v50, s18 +; VGPRRC-NEXT: v_mov_b32_e32 v51, s19 ; VGPRRC-NEXT: s_nop 7 ; VGPRRC-NEXT: s_nop 0 -; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[28:31], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[24:27], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[48:49], v[20:23], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[50:51], v[16:19], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[40:43], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[48:51], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: v_mov_b32_e32 v0, s20 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s21 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s22 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s23 -; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 -; VGPRRC-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v0, s12 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s13 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s14 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s15 -; VGPRRC-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_endpgm ; AGPR-LABEL: test_mfma_f32_32x32x16_f16: @@ -776,9 +776,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, < ; SDAG: ; %bb.0: ; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; SDAG-NEXT: v_mov_b64_e32 v[12:13], 48 -; SDAG-NEXT: v_mov_b64_e32 v[14:15], 32 -; SDAG-NEXT: v_mov_b64_e32 v[16:17], 16 +; SDAG-NEXT: v_mov_b64_e32 v[8:9], 48 +; SDAG-NEXT: v_mov_b64_e32 v[10:11], 32 +; SDAG-NEXT: v_mov_b64_e32 v[12:13], 16 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -800,42 +800,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, < ; SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; SDAG-NEXT: v_accvgpr_write_b32 a15, s23 -; SDAG-NEXT: v_mov_b64_e32 v[18:19], 0 -; SDAG-NEXT: v_mov_b32_e32 v8, s16 +; SDAG-NEXT: v_mov_b64_e32 v[14:15], 0 +; SDAG-NEXT: v_mov_b32_e32 v16, s16 ; SDAG-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1 ; SDAG-NEXT: v_mov_b32_e32 v0, s20 ; SDAG-NEXT: v_mov_b32_e32 v1, s21 ; SDAG-NEXT: v_mov_b32_e32 v2, s22 ; SDAG-NEXT: v_mov_b32_e32 v3, s23 -; SDAG-NEXT: v_mov_b32_e32 v9, s17 -; SDAG-NEXT: v_mov_b32_e32 v10, s18 -; SDAG-NEXT: v_mov_b32_e32 v11, s19 +; SDAG-NEXT: v_mov_b32_e32 v17, s17 +; SDAG-NEXT: v_mov_b32_e32 v18, s18 +; SDAG-NEXT: v_mov_b32_e32 v19, s19 ; SDAG-NEXT: s_nop 4 -; SDAG-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[10:11], v[16:19], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s8 ; SDAG-NEXT: v_mov_b32_e32 v1, s9 ; SDAG-NEXT: v_mov_b32_e32 v2, s10 ; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s12 ; SDAG-NEXT: v_mov_b32_e32 v1, s13 ; SDAG-NEXT: v_mov_b32_e32 v2, s14 ; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; @@ -900,9 +900,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, < ; HEURRC: ; %bb.0: ; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 48 -; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 32 -; HEURRC-NEXT: v_mov_b64_e32 v[16:17], 16 +; HEURRC-NEXT: v_mov_b64_e32 v[8:9], 48 +; HEURRC-NEXT: v_mov_b64_e32 v[10:11], 32 +; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 16 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) ; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -924,42 +924,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, < ; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 ; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 ; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 -; HEURRC-NEXT: v_mov_b64_e32 v[18:19], 0 -; HEURRC-NEXT: v_mov_b32_e32 v8, s16 +; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 0 +; HEURRC-NEXT: v_mov_b32_e32 v16, s16 ; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1 ; HEURRC-NEXT: v_mov_b32_e32 v0, s20 ; HEURRC-NEXT: v_mov_b32_e32 v1, s21 ; HEURRC-NEXT: v_mov_b32_e32 v2, s22 ; HEURRC-NEXT: v_mov_b32_e32 v3, s23 -; HEURRC-NEXT: v_mov_b32_e32 v9, s17 -; HEURRC-NEXT: v_mov_b32_e32 v10, s18 -; HEURRC-NEXT: v_mov_b32_e32 v11, s19 +; HEURRC-NEXT: v_mov_b32_e32 v17, s17 +; HEURRC-NEXT: v_mov_b32_e32 v18, s18 +; HEURRC-NEXT: v_mov_b32_e32 v19, s19 ; HEURRC-NEXT: s_nop 4 -; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[10:11], v[16:19], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s8 ; HEURRC-NEXT: v_mov_b32_e32 v1, s9 ; HEURRC-NEXT: v_mov_b32_e32 v2, s10 ; HEURRC-NEXT: v_mov_b32_e32 v3, s11 -; HEURRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s12 ; HEURRC-NEXT: v_mov_b32_e32 v1, s13 ; HEURRC-NEXT: v_mov_b32_e32 v2, s14 ; HEURRC-NEXT: v_mov_b32_e32 v3, s15 -; HEURRC-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_endpgm ; @@ -967,9 +967,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, < ; VGPRRC: ; %bb.0: ; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 48 -; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 32 -; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], 16 +; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], 48 +; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], 32 +; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 16 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) ; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27] ; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25] @@ -983,43 +983,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, < ; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] ; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] ; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] -; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], 0 -; VGPRRC-NEXT: v_mov_b32_e32 v40, s16 +; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 0 +; VGPRRC-NEXT: v_mov_b32_e32 v48, s16 ; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1 -; VGPRRC-NEXT: v_mov_b32_e32 v41, s17 -; VGPRRC-NEXT: v_mov_b32_e32 v42, s18 -; VGPRRC-NEXT: v_mov_b32_e32 v43, s19 +; VGPRRC-NEXT: v_mov_b32_e32 v49, s17 +; VGPRRC-NEXT: v_mov_b32_e32 v50, s18 +; VGPRRC-NEXT: v_mov_b32_e32 v51, s19 ; VGPRRC-NEXT: s_nop 7 ; VGPRRC-NEXT: s_nop 0 -; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[28:31], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[24:27], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[48:49], v[20:23], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[50:51], v[16:19], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[40:43], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[48:51], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: v_mov_b32_e32 v0, s20 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s21 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s22 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s23 -; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 -; VGPRRC-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v0, s12 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s13 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s14 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s15 -; VGPRRC-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_endpgm ; AGPR-LABEL: test_mfma_f32_32x32x16_f16__flags: @@ -1505,7 +1505,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0, ; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; SDAG-NEXT: v_mov_b32_e32 v12, 0 +; SDAG-NEXT: v_mov_b32_e32 v8, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -1527,40 +1527,40 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0, ; SDAG-NEXT: v_accvgpr_write_b32 a18, s10 ; SDAG-NEXT: v_accvgpr_write_b32 a17, s9 ; SDAG-NEXT: v_accvgpr_write_b32 a16, s8 -; SDAG-NEXT: v_mov_b32_e32 v8, s20 -; SDAG-NEXT: v_mov_b32_e32 v9, s21 +; SDAG-NEXT: v_mov_b32_e32 v10, s20 +; SDAG-NEXT: v_mov_b32_e32 v11, s21 ; SDAG-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31] -; SDAG-NEXT: v_mov_b32_e32 v10, s22 -; SDAG-NEXT: v_mov_b32_e32 v11, s23 +; SDAG-NEXT: v_mov_b32_e32 v12, s22 +; SDAG-NEXT: v_mov_b32_e32 v13, s23 ; SDAG-NEXT: v_mov_b32_e32 v0, s16 ; SDAG-NEXT: v_mov_b32_e32 v1, s17 ; SDAG-NEXT: v_mov_b32_e32 v2, s18 ; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, v[10:13], s[0:1] offset:48 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s12 ; SDAG-NEXT: v_mov_b32_e32 v1, s13 ; SDAG-NEXT: v_mov_b32_e32 v2, s14 ; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s8 ; SDAG-NEXT: v_mov_b32_e32 v1, s9 ; SDAG-NEXT: v_mov_b32_e32 v2, s10 ; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; @@ -1623,7 +1623,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0, ; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 ; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; HEURRC-NEXT: v_mov_b32_e32 v12, 0 +; HEURRC-NEXT: v_mov_b32_e32 v8, 0 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) ; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -1645,40 +1645,40 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0, ; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10 ; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9 ; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8 -; HEURRC-NEXT: v_mov_b32_e32 v8, s20 -; HEURRC-NEXT: v_mov_b32_e32 v9, s21 +; HEURRC-NEXT: v_mov_b32_e32 v10, s20 +; HEURRC-NEXT: v_mov_b32_e32 v11, s21 ; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31] -; HEURRC-NEXT: v_mov_b32_e32 v10, s22 -; HEURRC-NEXT: v_mov_b32_e32 v11, s23 +; HEURRC-NEXT: v_mov_b32_e32 v12, s22 +; HEURRC-NEXT: v_mov_b32_e32 v13, s23 ; HEURRC-NEXT: v_mov_b32_e32 v0, s16 ; HEURRC-NEXT: v_mov_b32_e32 v1, s17 ; HEURRC-NEXT: v_mov_b32_e32 v2, s18 ; HEURRC-NEXT: v_mov_b32_e32 v3, s19 -; HEURRC-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, v[10:13], s[0:1] offset:48 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s12 ; HEURRC-NEXT: v_mov_b32_e32 v1, s13 ; HEURRC-NEXT: v_mov_b32_e32 v2, s14 ; HEURRC-NEXT: v_mov_b32_e32 v3, s15 -; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s8 ; HEURRC-NEXT: v_mov_b32_e32 v1, s9 ; HEURRC-NEXT: v_mov_b32_e32 v2, s10 ; HEURRC-NEXT: v_mov_b32_e32 v3, s11 -; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_endpgm ; @@ -1687,7 +1687,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0, ; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 ; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; VGPRRC-NEXT: v_mov_b32_e32 v44, 0 +; VGPRRC-NEXT: v_mov_b32_e32 v40, 0 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) ; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27] ; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25] @@ -1701,41 +1701,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0, ; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13] ; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11] ; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9] -; VGPRRC-NEXT: v_mov_b32_e32 v40, s20 -; VGPRRC-NEXT: v_mov_b32_e32 v41, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v42, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v43, s21 ; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] -; VGPRRC-NEXT: v_mov_b32_e32 v42, s22 -; VGPRRC-NEXT: v_mov_b32_e32 v43, s23 -; VGPRRC-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: v_mov_b32_e32 v44, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v45, s23 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[42:45], s[0:1] offset:48 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 2 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s16 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s17 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s18 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s19 -; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s12 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s13 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s14 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s15 -; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s8 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s9 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s10 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s11 -; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_endpgm ; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd: @@ -1869,7 +1869,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half> ; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; SDAG-NEXT: v_mov_b32_e32 v12, 0 +; SDAG-NEXT: v_mov_b32_e32 v8, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -1891,40 +1891,40 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half> ; SDAG-NEXT: v_accvgpr_write_b32 a18, s10 ; SDAG-NEXT: v_accvgpr_write_b32 a17, s9 ; SDAG-NEXT: v_accvgpr_write_b32 a16, s8 -; SDAG-NEXT: v_mov_b32_e32 v8, s20 -; SDAG-NEXT: v_mov_b32_e32 v9, s21 +; SDAG-NEXT: v_mov_b32_e32 v10, s20 +; SDAG-NEXT: v_mov_b32_e32 v11, s21 ; SDAG-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:3 -; SDAG-NEXT: v_mov_b32_e32 v10, s22 -; SDAG-NEXT: v_mov_b32_e32 v11, s23 +; SDAG-NEXT: v_mov_b32_e32 v12, s22 +; SDAG-NEXT: v_mov_b32_e32 v13, s23 ; SDAG-NEXT: v_mov_b32_e32 v0, s16 ; SDAG-NEXT: v_mov_b32_e32 v1, s17 ; SDAG-NEXT: v_mov_b32_e32 v2, s18 ; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, v[10:13], s[0:1] offset:48 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s12 ; SDAG-NEXT: v_mov_b32_e32 v1, s13 ; SDAG-NEXT: v_mov_b32_e32 v2, s14 ; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s8 ; SDAG-NEXT: v_mov_b32_e32 v1, s9 ; SDAG-NEXT: v_mov_b32_e32 v2, s10 ; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; @@ -1987,7 +1987,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half> ; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 ; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; HEURRC-NEXT: v_mov_b32_e32 v12, 0 +; HEURRC-NEXT: v_mov_b32_e32 v8, 0 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) ; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25] ; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27] @@ -2009,40 +2009,40 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half> ; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10 ; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9 ; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8 -; HEURRC-NEXT: v_mov_b32_e32 v8, s20 -; HEURRC-NEXT: v_mov_b32_e32 v9, s21 +; HEURRC-NEXT: v_mov_b32_e32 v10, s20 +; HEURRC-NEXT: v_mov_b32_e32 v11, s21 ; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:3 -; HEURRC-NEXT: v_mov_b32_e32 v10, s22 -; HEURRC-NEXT: v_mov_b32_e32 v11, s23 +; HEURRC-NEXT: v_mov_b32_e32 v12, s22 +; HEURRC-NEXT: v_mov_b32_e32 v13, s23 ; HEURRC-NEXT: v_mov_b32_e32 v0, s16 ; HEURRC-NEXT: v_mov_b32_e32 v1, s17 ; HEURRC-NEXT: v_mov_b32_e32 v2, s18 ; HEURRC-NEXT: v_mov_b32_e32 v3, s19 -; HEURRC-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, v[10:13], s[0:1] offset:48 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s12 ; HEURRC-NEXT: v_mov_b32_e32 v1, s13 ; HEURRC-NEXT: v_mov_b32_e32 v2, s14 ; HEURRC-NEXT: v_mov_b32_e32 v3, s15 -; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s8 ; HEURRC-NEXT: v_mov_b32_e32 v1, s9 ; HEURRC-NEXT: v_mov_b32_e32 v2, s10 ; HEURRC-NEXT: v_mov_b32_e32 v3, s11 -; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_endpgm ; @@ -2051,7 +2051,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half> ; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 ; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; VGPRRC-NEXT: v_mov_b32_e32 v44, 0 +; VGPRRC-NEXT: v_mov_b32_e32 v40, 0 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) ; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27] ; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25] @@ -2065,41 +2065,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half> ; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13] ; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11] ; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9] -; VGPRRC-NEXT: v_mov_b32_e32 v40, s20 -; VGPRRC-NEXT: v_mov_b32_e32 v41, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v42, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v43, s21 ; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3 -; VGPRRC-NEXT: v_mov_b32_e32 v42, s22 -; VGPRRC-NEXT: v_mov_b32_e32 v43, s23 -; VGPRRC-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: v_mov_b32_e32 v44, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v45, s23 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[42:45], s[0:1] offset:48 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 2 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s16 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s17 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s18 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s19 -; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s12 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s13 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s14 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s15 -; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s8 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s9 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s10 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s11 -; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_endpgm ; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags: @@ -2781,24 +2781,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa ; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 -; SDAG-NEXT: v_mov_b32_e32 v8, 0 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: v_mov_b32_e32 v4, s12 -; SDAG-NEXT: v_mov_b32_e32 v5, s13 -; SDAG-NEXT: v_mov_b32_e32 v6, s14 -; SDAG-NEXT: v_mov_b32_e32 v7, s15 +; SDAG-NEXT: v_mov_b32_e32 v2, s8 +; SDAG-NEXT: v_mov_b32_e32 v3, s9 +; SDAG-NEXT: v_mov_b32_e32 v4, s10 +; SDAG-NEXT: v_mov_b32_e32 v5, s11 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] +; SDAG-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[2:5], v[6:9], a[0:3] ; SDAG-NEXT: s_nop 7 -; SDAG-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd: @@ -2827,24 +2827,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa ; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 -; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, 0 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) -; HEURRC-NEXT: v_mov_b32_e32 v0, s8 -; HEURRC-NEXT: v_mov_b32_e32 v1, s9 -; HEURRC-NEXT: v_mov_b32_e32 v2, s10 -; HEURRC-NEXT: v_mov_b32_e32 v3, s11 -; HEURRC-NEXT: v_mov_b32_e32 v4, s12 -; HEURRC-NEXT: v_mov_b32_e32 v5, s13 -; HEURRC-NEXT: v_mov_b32_e32 v6, s14 -; HEURRC-NEXT: v_mov_b32_e32 v7, s15 +; HEURRC-NEXT: v_mov_b32_e32 v2, s8 +; HEURRC-NEXT: v_mov_b32_e32 v3, s9 +; HEURRC-NEXT: v_mov_b32_e32 v4, s10 +; HEURRC-NEXT: v_mov_b32_e32 v5, s11 +; HEURRC-NEXT: v_mov_b32_e32 v6, s12 +; HEURRC-NEXT: v_mov_b32_e32 v7, s13 +; HEURRC-NEXT: v_mov_b32_e32 v8, s14 +; HEURRC-NEXT: v_mov_b32_e32 v9, s15 ; HEURRC-NEXT: v_accvgpr_write_b32 a0, s0 ; HEURRC-NEXT: v_accvgpr_write_b32 a1, s1 ; HEURRC-NEXT: v_accvgpr_write_b32 a2, s2 ; HEURRC-NEXT: v_accvgpr_write_b32 a3, s3 ; HEURRC-NEXT: s_nop 1 -; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] +; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[2:5], v[6:9], a[0:3] ; HEURRC-NEXT: s_nop 7 -; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; HEURRC-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; HEURRC-NEXT: s_endpgm ; ; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd: @@ -2852,24 +2852,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa ; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 -; VGPRRC-NEXT: v_mov_b32_e32 v12, 0 +; VGPRRC-NEXT: v_mov_b32_e32 v4, 0 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) -; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 -; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 -; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 -; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 -; VGPRRC-NEXT: v_mov_b32_e32 v4, s12 -; VGPRRC-NEXT: v_mov_b32_e32 v5, s13 -; VGPRRC-NEXT: v_mov_b32_e32 v6, s14 -; VGPRRC-NEXT: v_mov_b32_e32 v7, s15 -; VGPRRC-NEXT: v_mov_b32_e32 v8, s0 -; VGPRRC-NEXT: v_mov_b32_e32 v9, s1 -; VGPRRC-NEXT: v_mov_b32_e32 v10, s2 -; VGPRRC-NEXT: v_mov_b32_e32 v11, s3 +; VGPRRC-NEXT: v_mov_b32_e32 v6, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v7, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v8, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v9, s11 +; VGPRRC-NEXT: v_mov_b32_e32 v10, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v11, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v12, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v13, s15 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s0 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s1 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s2 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s3 ; VGPRRC-NEXT: s_nop 1 -; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3] ; VGPRRC-NEXT: s_nop 7 -; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPRRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] ; VGPRRC-NEXT: s_endpgm ; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd: ; AGPR: ; %bb.0: @@ -2930,24 +2930,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr ; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 -; SDAG-NEXT: v_mov_b32_e32 v8, 0 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: v_mov_b32_e32 v4, s12 -; SDAG-NEXT: v_mov_b32_e32 v5, s13 -; SDAG-NEXT: v_mov_b32_e32 v6, s14 -; SDAG-NEXT: v_mov_b32_e32 v7, s15 +; SDAG-NEXT: v_mov_b32_e32 v2, s8 +; SDAG-NEXT: v_mov_b32_e32 v3, s9 +; SDAG-NEXT: v_mov_b32_e32 v4, s10 +; SDAG-NEXT: v_mov_b32_e32 v5, s11 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1 +; SDAG-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[2:5], v[6:9], a[0:3] cbsz:3 abid:2 blgp:1 ; SDAG-NEXT: s_nop 7 -; SDAG-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags: @@ -2976,24 +2976,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr ; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 -; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, 0 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) -; HEURRC-NEXT: v_mov_b32_e32 v0, s8 -; HEURRC-NEXT: v_mov_b32_e32 v1, s9 -; HEURRC-NEXT: v_mov_b32_e32 v2, s10 -; HEURRC-NEXT: v_mov_b32_e32 v3, s11 -; HEURRC-NEXT: v_mov_b32_e32 v4, s12 -; HEURRC-NEXT: v_mov_b32_e32 v5, s13 -; HEURRC-NEXT: v_mov_b32_e32 v6, s14 -; HEURRC-NEXT: v_mov_b32_e32 v7, s15 +; HEURRC-NEXT: v_mov_b32_e32 v2, s8 +; HEURRC-NEXT: v_mov_b32_e32 v3, s9 +; HEURRC-NEXT: v_mov_b32_e32 v4, s10 +; HEURRC-NEXT: v_mov_b32_e32 v5, s11 +; HEURRC-NEXT: v_mov_b32_e32 v6, s12 +; HEURRC-NEXT: v_mov_b32_e32 v7, s13 +; HEURRC-NEXT: v_mov_b32_e32 v8, s14 +; HEURRC-NEXT: v_mov_b32_e32 v9, s15 ; HEURRC-NEXT: v_accvgpr_write_b32 a0, s0 ; HEURRC-NEXT: v_accvgpr_write_b32 a1, s1 ; HEURRC-NEXT: v_accvgpr_write_b32 a2, s2 ; HEURRC-NEXT: v_accvgpr_write_b32 a3, s3 ; HEURRC-NEXT: s_nop 1 -; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1 +; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[2:5], v[6:9], a[0:3] cbsz:3 abid:2 blgp:1 ; HEURRC-NEXT: s_nop 7 -; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; HEURRC-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; HEURRC-NEXT: s_endpgm ; ; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags: @@ -3001,24 +3001,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr ; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 -; VGPRRC-NEXT: v_mov_b32_e32 v12, 0 +; VGPRRC-NEXT: v_mov_b32_e32 v4, 0 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) -; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 -; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 -; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 -; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 -; VGPRRC-NEXT: v_mov_b32_e32 v4, s12 -; VGPRRC-NEXT: v_mov_b32_e32 v5, s13 -; VGPRRC-NEXT: v_mov_b32_e32 v6, s14 -; VGPRRC-NEXT: v_mov_b32_e32 v7, s15 -; VGPRRC-NEXT: v_mov_b32_e32 v8, s0 -; VGPRRC-NEXT: v_mov_b32_e32 v9, s1 -; VGPRRC-NEXT: v_mov_b32_e32 v10, s2 -; VGPRRC-NEXT: v_mov_b32_e32 v11, s3 +; VGPRRC-NEXT: v_mov_b32_e32 v6, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v7, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v8, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v9, s11 +; VGPRRC-NEXT: v_mov_b32_e32 v10, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v11, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v12, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v13, s15 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s0 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s1 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s2 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s3 ; VGPRRC-NEXT: s_nop 1 -; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1 +; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1 ; VGPRRC-NEXT: s_nop 7 -; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPRRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] ; VGPRRC-NEXT: s_endpgm ; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags: ; AGPR: ; %bb.0: @@ -3084,19 +3084,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32> ; SDAG: ; %bb.0: ; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; SDAG-NEXT: v_mov_b64_e32 v[8:9], 48 -; SDAG-NEXT: v_mov_b64_e32 v[10:11], 32 -; SDAG-NEXT: v_mov_b64_e32 v[12:13], 16 +; SDAG-NEXT: v_mov_b64_e32 v[0:1], 48 +; SDAG-NEXT: v_mov_b64_e32 v[2:3], 32 +; SDAG-NEXT: v_mov_b64_e32 v[4:5], 16 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s24 -; SDAG-NEXT: v_mov_b32_e32 v1, s25 -; SDAG-NEXT: v_mov_b32_e32 v2, s26 -; SDAG-NEXT: v_mov_b32_e32 v3, s27 +; SDAG-NEXT: v_mov_b32_e32 v8, s24 +; SDAG-NEXT: v_mov_b32_e32 v9, s25 +; SDAG-NEXT: v_mov_b32_e32 v10, s26 +; SDAG-NEXT: v_mov_b32_e32 v11, s27 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s8 -; SDAG-NEXT: v_mov_b32_e32 v4, s28 -; SDAG-NEXT: v_mov_b32_e32 v5, s29 -; SDAG-NEXT: v_mov_b32_e32 v6, s30 -; SDAG-NEXT: v_mov_b32_e32 v7, s31 +; SDAG-NEXT: v_mov_b32_e32 v12, s28 +; SDAG-NEXT: v_mov_b32_e32 v13, s29 +; SDAG-NEXT: v_mov_b32_e32 v14, s30 +; SDAG-NEXT: v_mov_b32_e32 v15, s31 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s11 @@ -3112,44 +3112,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32> ; SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; SDAG-NEXT: v_accvgpr_write_b32 a15, s23 -; SDAG-NEXT: v_mov_b64_e32 v[14:15], 0 +; SDAG-NEXT: v_mov_b64_e32 v[6:7], 0 ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 +; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15] +; SDAG-NEXT: v_mov_b32_e32 v8, s16 +; SDAG-NEXT: v_mov_b32_e32 v9, s17 +; SDAG-NEXT: v_mov_b32_e32 v10, s18 +; SDAG-NEXT: v_mov_b32_e32 v11, s19 ; SDAG-NEXT: s_nop 7 -; SDAG-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s20 -; SDAG-NEXT: v_mov_b32_e32 v1, s21 -; SDAG-NEXT: v_mov_b32_e32 v2, s22 -; SDAG-NEXT: v_mov_b32_e32 v3, s23 -; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v2, s10 +; SDAG-NEXT: v_mov_b32_e32 v3, s11 +; SDAG-NEXT: v_mov_b32_e32 v8, s20 +; SDAG-NEXT: v_mov_b32_e32 v9, s21 +; SDAG-NEXT: v_mov_b32_e32 v10, s22 +; SDAG-NEXT: v_mov_b32_e32 v11, s23 +; SDAG-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s8 ; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s12 ; SDAG-NEXT: v_mov_b32_e32 v1, s13 ; SDAG-NEXT: v_mov_b32_e32 v2, s14 ; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; @@ -3214,19 +3212,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32> ; HEURRC: ; %bb.0: ; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; HEURRC-NEXT: v_mov_b64_e32 v[8:9], 48 -; HEURRC-NEXT: v_mov_b64_e32 v[10:11], 32 -; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 16 +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48 +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32 +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) -; HEURRC-NEXT: v_mov_b32_e32 v0, s24 -; HEURRC-NEXT: v_mov_b32_e32 v1, s25 -; HEURRC-NEXT: v_mov_b32_e32 v2, s26 -; HEURRC-NEXT: v_mov_b32_e32 v3, s27 +; HEURRC-NEXT: v_mov_b32_e32 v8, s24 +; HEURRC-NEXT: v_mov_b32_e32 v9, s25 +; HEURRC-NEXT: v_mov_b32_e32 v10, s26 +; HEURRC-NEXT: v_mov_b32_e32 v11, s27 ; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8 -; HEURRC-NEXT: v_mov_b32_e32 v4, s28 -; HEURRC-NEXT: v_mov_b32_e32 v5, s29 -; HEURRC-NEXT: v_mov_b32_e32 v6, s30 -; HEURRC-NEXT: v_mov_b32_e32 v7, s31 +; HEURRC-NEXT: v_mov_b32_e32 v12, s28 +; HEURRC-NEXT: v_mov_b32_e32 v13, s29 +; HEURRC-NEXT: v_mov_b32_e32 v14, s30 +; HEURRC-NEXT: v_mov_b32_e32 v15, s31 ; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9 ; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10 ; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11 @@ -3242,44 +3240,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32> ; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 ; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 ; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 -; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 0 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0 ; HEURRC-NEXT: s_nop 0 -; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] -; HEURRC-NEXT: v_mov_b32_e32 v0, s16 -; HEURRC-NEXT: v_mov_b32_e32 v1, s17 -; HEURRC-NEXT: v_mov_b32_e32 v2, s18 -; HEURRC-NEXT: v_mov_b32_e32 v3, s19 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15] +; HEURRC-NEXT: v_mov_b32_e32 v8, s16 +; HEURRC-NEXT: v_mov_b32_e32 v9, s17 +; HEURRC-NEXT: v_mov_b32_e32 v10, s18 +; HEURRC-NEXT: v_mov_b32_e32 v11, s19 ; HEURRC-NEXT: s_nop 7 -; HEURRC-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: s_nop 0 -; HEURRC-NEXT: v_mov_b32_e32 v0, s20 -; HEURRC-NEXT: v_mov_b32_e32 v1, s21 -; HEURRC-NEXT: v_mov_b32_e32 v2, s22 -; HEURRC-NEXT: v_mov_b32_e32 v3, s23 -; HEURRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: v_mov_b32_e32 v8, s20 +; HEURRC-NEXT: v_mov_b32_e32 v9, s21 +; HEURRC-NEXT: v_mov_b32_e32 v10, s22 +; HEURRC-NEXT: v_mov_b32_e32 v11, s23 +; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s8 ; HEURRC-NEXT: v_mov_b32_e32 v1, s9 -; HEURRC-NEXT: v_mov_b32_e32 v2, s10 -; HEURRC-NEXT: v_mov_b32_e32 v3, s11 -; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s12 ; HEURRC-NEXT: v_mov_b32_e32 v1, s13 ; HEURRC-NEXT: v_mov_b32_e32 v2, s14 ; HEURRC-NEXT: v_mov_b32_e32 v3, s15 -; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_endpgm ; @@ -3287,19 +3283,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32> ; VGPRRC: ; %bb.0: ; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], 48 -; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], 32 -; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 16 +; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48 +; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32 +; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) -; VGPRRC-NEXT: v_mov_b32_e32 v32, s24 -; VGPRRC-NEXT: v_mov_b32_e32 v33, s25 -; VGPRRC-NEXT: v_mov_b32_e32 v34, s26 -; VGPRRC-NEXT: v_mov_b32_e32 v35, s27 +; VGPRRC-NEXT: v_mov_b32_e32 v40, s24 +; VGPRRC-NEXT: v_mov_b32_e32 v41, s25 +; VGPRRC-NEXT: v_mov_b32_e32 v42, s26 +; VGPRRC-NEXT: v_mov_b32_e32 v43, s27 ; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] -; VGPRRC-NEXT: v_mov_b32_e32 v36, s28 -; VGPRRC-NEXT: v_mov_b32_e32 v37, s29 -; VGPRRC-NEXT: v_mov_b32_e32 v38, s30 -; VGPRRC-NEXT: v_mov_b32_e32 v39, s31 +; VGPRRC-NEXT: v_mov_b32_e32 v44, s28 +; VGPRRC-NEXT: v_mov_b32_e32 v45, s29 +; VGPRRC-NEXT: v_mov_b32_e32 v46, s30 +; VGPRRC-NEXT: v_mov_b32_e32 v47, s31 ; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] ; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] @@ -3307,45 +3303,45 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32> ; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] ; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] ; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] -; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 0 +; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0 ; VGPRRC-NEXT: s_nop 0 -; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15] ; VGPRRC-NEXT: s_nop 7 ; VGPRRC-NEXT: s_nop 3 -; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: v_mov_b32_e32 v0, s16 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s17 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s18 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s19 -; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v0, s20 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s21 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s22 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s23 -; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 -; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v0, s12 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s13 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s14 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s15 -; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_endpgm ; AGPR-LABEL: test_mfma_i32_32x32x32_i8: @@ -3496,19 +3492,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4 ; SDAG: ; %bb.0: ; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; SDAG-NEXT: v_mov_b64_e32 v[8:9], 48 -; SDAG-NEXT: v_mov_b64_e32 v[10:11], 32 -; SDAG-NEXT: v_mov_b64_e32 v[12:13], 16 +; SDAG-NEXT: v_mov_b64_e32 v[0:1], 48 +; SDAG-NEXT: v_mov_b64_e32 v[2:3], 32 +; SDAG-NEXT: v_mov_b64_e32 v[4:5], 16 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s24 -; SDAG-NEXT: v_mov_b32_e32 v1, s25 -; SDAG-NEXT: v_mov_b32_e32 v2, s26 -; SDAG-NEXT: v_mov_b32_e32 v3, s27 +; SDAG-NEXT: v_mov_b32_e32 v8, s24 +; SDAG-NEXT: v_mov_b32_e32 v9, s25 +; SDAG-NEXT: v_mov_b32_e32 v10, s26 +; SDAG-NEXT: v_mov_b32_e32 v11, s27 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s8 -; SDAG-NEXT: v_mov_b32_e32 v4, s28 -; SDAG-NEXT: v_mov_b32_e32 v5, s29 -; SDAG-NEXT: v_mov_b32_e32 v6, s30 -; SDAG-NEXT: v_mov_b32_e32 v7, s31 +; SDAG-NEXT: v_mov_b32_e32 v12, s28 +; SDAG-NEXT: v_mov_b32_e32 v13, s29 +; SDAG-NEXT: v_mov_b32_e32 v14, s30 +; SDAG-NEXT: v_mov_b32_e32 v15, s31 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s11 @@ -3524,44 +3520,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4 ; SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; SDAG-NEXT: v_accvgpr_write_b32 a15, s23 -; SDAG-NEXT: v_mov_b64_e32 v[14:15], 0 +; SDAG-NEXT: v_mov_b64_e32 v[6:7], 0 ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 +; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1 +; SDAG-NEXT: v_mov_b32_e32 v8, s16 +; SDAG-NEXT: v_mov_b32_e32 v9, s17 +; SDAG-NEXT: v_mov_b32_e32 v10, s18 +; SDAG-NEXT: v_mov_b32_e32 v11, s19 ; SDAG-NEXT: s_nop 7 -; SDAG-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s20 -; SDAG-NEXT: v_mov_b32_e32 v1, s21 -; SDAG-NEXT: v_mov_b32_e32 v2, s22 -; SDAG-NEXT: v_mov_b32_e32 v3, s23 -; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v2, s10 +; SDAG-NEXT: v_mov_b32_e32 v3, s11 +; SDAG-NEXT: v_mov_b32_e32 v8, s20 +; SDAG-NEXT: v_mov_b32_e32 v9, s21 +; SDAG-NEXT: v_mov_b32_e32 v10, s22 +; SDAG-NEXT: v_mov_b32_e32 v11, s23 +; SDAG-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s8 ; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 ; SDAG-NEXT: v_mov_b32_e32 v0, s12 ; SDAG-NEXT: v_mov_b32_e32 v1, s13 ; SDAG-NEXT: v_mov_b32_e32 v2, s14 ; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; @@ -3626,19 +3620,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4 ; HEURRC: ; %bb.0: ; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; HEURRC-NEXT: v_mov_b64_e32 v[8:9], 48 -; HEURRC-NEXT: v_mov_b64_e32 v[10:11], 32 -; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 16 +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48 +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32 +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) -; HEURRC-NEXT: v_mov_b32_e32 v0, s24 -; HEURRC-NEXT: v_mov_b32_e32 v1, s25 -; HEURRC-NEXT: v_mov_b32_e32 v2, s26 -; HEURRC-NEXT: v_mov_b32_e32 v3, s27 +; HEURRC-NEXT: v_mov_b32_e32 v8, s24 +; HEURRC-NEXT: v_mov_b32_e32 v9, s25 +; HEURRC-NEXT: v_mov_b32_e32 v10, s26 +; HEURRC-NEXT: v_mov_b32_e32 v11, s27 ; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8 -; HEURRC-NEXT: v_mov_b32_e32 v4, s28 -; HEURRC-NEXT: v_mov_b32_e32 v5, s29 -; HEURRC-NEXT: v_mov_b32_e32 v6, s30 -; HEURRC-NEXT: v_mov_b32_e32 v7, s31 +; HEURRC-NEXT: v_mov_b32_e32 v12, s28 +; HEURRC-NEXT: v_mov_b32_e32 v13, s29 +; HEURRC-NEXT: v_mov_b32_e32 v14, s30 +; HEURRC-NEXT: v_mov_b32_e32 v15, s31 ; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9 ; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10 ; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11 @@ -3654,44 +3648,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4 ; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 ; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 ; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 -; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 0 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0 ; HEURRC-NEXT: s_nop 0 -; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1 -; HEURRC-NEXT: v_mov_b32_e32 v0, s16 -; HEURRC-NEXT: v_mov_b32_e32 v1, s17 -; HEURRC-NEXT: v_mov_b32_e32 v2, s18 -; HEURRC-NEXT: v_mov_b32_e32 v3, s19 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1 +; HEURRC-NEXT: v_mov_b32_e32 v8, s16 +; HEURRC-NEXT: v_mov_b32_e32 v9, s17 +; HEURRC-NEXT: v_mov_b32_e32 v10, s18 +; HEURRC-NEXT: v_mov_b32_e32 v11, s19 ; HEURRC-NEXT: s_nop 7 -; HEURRC-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: s_nop 0 -; HEURRC-NEXT: v_mov_b32_e32 v0, s20 -; HEURRC-NEXT: v_mov_b32_e32 v1, s21 -; HEURRC-NEXT: v_mov_b32_e32 v2, s22 -; HEURRC-NEXT: v_mov_b32_e32 v3, s23 -; HEURRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: v_mov_b32_e32 v8, s20 +; HEURRC-NEXT: v_mov_b32_e32 v9, s21 +; HEURRC-NEXT: v_mov_b32_e32 v10, s22 +; HEURRC-NEXT: v_mov_b32_e32 v11, s23 +; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s8 ; HEURRC-NEXT: v_mov_b32_e32 v1, s9 -; HEURRC-NEXT: v_mov_b32_e32 v2, s10 -; HEURRC-NEXT: v_mov_b32_e32 v3, s11 -; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 ; HEURRC-NEXT: v_mov_b32_e32 v0, s12 ; HEURRC-NEXT: v_mov_b32_e32 v1, s13 ; HEURRC-NEXT: v_mov_b32_e32 v2, s14 ; HEURRC-NEXT: v_mov_b32_e32 v3, s15 -; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_endpgm ; @@ -3699,19 +3691,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4 ; VGPRRC: ; %bb.0: ; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], 48 -; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], 32 -; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 16 +; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48 +; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32 +; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) -; VGPRRC-NEXT: v_mov_b32_e32 v32, s24 -; VGPRRC-NEXT: v_mov_b32_e32 v33, s25 -; VGPRRC-NEXT: v_mov_b32_e32 v34, s26 -; VGPRRC-NEXT: v_mov_b32_e32 v35, s27 +; VGPRRC-NEXT: v_mov_b32_e32 v40, s24 +; VGPRRC-NEXT: v_mov_b32_e32 v41, s25 +; VGPRRC-NEXT: v_mov_b32_e32 v42, s26 +; VGPRRC-NEXT: v_mov_b32_e32 v43, s27 ; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] -; VGPRRC-NEXT: v_mov_b32_e32 v36, s28 -; VGPRRC-NEXT: v_mov_b32_e32 v37, s29 -; VGPRRC-NEXT: v_mov_b32_e32 v38, s30 -; VGPRRC-NEXT: v_mov_b32_e32 v39, s31 +; VGPRRC-NEXT: v_mov_b32_e32 v44, s28 +; VGPRRC-NEXT: v_mov_b32_e32 v45, s29 +; VGPRRC-NEXT: v_mov_b32_e32 v46, s30 +; VGPRRC-NEXT: v_mov_b32_e32 v47, s31 ; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] ; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] @@ -3719,45 +3711,45 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4 ; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] ; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] ; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] -; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 0 +; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0 ; VGPRRC-NEXT: s_nop 0 -; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1 +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1 ; VGPRRC-NEXT: s_nop 7 ; VGPRRC-NEXT: s_nop 3 -; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: v_mov_b32_e32 v0, s16 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s17 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s18 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s19 -; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v0, s20 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s21 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s22 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s23 -; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 -; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v0, s12 ; VGPRRC-NEXT: v_mov_b32_e32 v1, s13 ; VGPRRC-NEXT: v_mov_b32_e32 v2, s14 ; VGPRRC-NEXT: v_mov_b32_e32 v3, s15 -; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_endpgm ; AGPR-LABEL: test_mfma_i32_32x32x32_i8__flags: @@ -4254,17 +4246,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 ; SDAG: ; %bb.0: ; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 ; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; SDAG-NEXT: v_mov_b32_e32 v8, 0 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s20 -; SDAG-NEXT: v_mov_b32_e32 v1, s21 -; SDAG-NEXT: v_mov_b32_e32 v2, s22 -; SDAG-NEXT: v_mov_b32_e32 v3, s23 +; SDAG-NEXT: v_mov_b32_e32 v2, s20 +; SDAG-NEXT: v_mov_b32_e32 v3, s21 +; SDAG-NEXT: v_mov_b32_e32 v4, s22 +; SDAG-NEXT: v_mov_b32_e32 v5, s23 ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; SDAG-NEXT: v_mov_b32_e32 v4, s24 -; SDAG-NEXT: v_mov_b32_e32 v5, s25 -; SDAG-NEXT: v_mov_b32_e32 v6, s26 -; SDAG-NEXT: v_mov_b32_e32 v7, s27 +; SDAG-NEXT: v_mov_b32_e32 v6, s24 +; SDAG-NEXT: v_mov_b32_e32 v7, s25 +; SDAG-NEXT: v_mov_b32_e32 v8, s26 +; SDAG-NEXT: v_mov_b32_e32 v9, s27 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_accvgpr_write_b32 a31, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a30, s22 @@ -4283,41 +4275,41 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 ; SDAG-NEXT: v_accvgpr_write_b32 a17, s9 ; SDAG-NEXT: v_accvgpr_write_b32 a16, s8 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] -; SDAG-NEXT: v_mov_b32_e32 v0, s20 -; SDAG-NEXT: v_mov_b32_e32 v1, s21 -; SDAG-NEXT: v_mov_b32_e32 v2, s22 -; SDAG-NEXT: v_mov_b32_e32 v3, s23 -; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc1 +; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[2:5], v[6:9], a[16:31] +; SDAG-NEXT: v_mov_b32_e32 v2, s20 +; SDAG-NEXT: v_mov_b32_e32 v3, s21 +; SDAG-NEXT: v_mov_b32_e32 v4, s22 +; SDAG-NEXT: v_mov_b32_e32 v5, s23 +; SDAG-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:48 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v2, s16 +; SDAG-NEXT: v_mov_b32_e32 v3, s17 +; SDAG-NEXT: v_mov_b32_e32 v4, s18 +; SDAG-NEXT: v_mov_b32_e32 v5, s19 +; SDAG-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:32 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v2, s12 +; SDAG-NEXT: v_mov_b32_e32 v3, s13 +; SDAG-NEXT: v_mov_b32_e32 v4, s14 +; SDAG-NEXT: v_mov_b32_e32 v5, s15 +; SDAG-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:16 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v2, s8 +; SDAG-NEXT: v_mov_b32_e32 v3, s9 +; SDAG-NEXT: v_mov_b32_e32 v4, s10 +; SDAG-NEXT: v_mov_b32_e32 v5, s11 +; SDAG-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; @@ -4379,17 +4371,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 ; HEURRC: ; %bb.0: ; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 ; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, 0 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) -; HEURRC-NEXT: v_mov_b32_e32 v0, s20 -; HEURRC-NEXT: v_mov_b32_e32 v1, s21 -; HEURRC-NEXT: v_mov_b32_e32 v2, s22 -; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: v_mov_b32_e32 v2, s20 +; HEURRC-NEXT: v_mov_b32_e32 v3, s21 +; HEURRC-NEXT: v_mov_b32_e32 v4, s22 +; HEURRC-NEXT: v_mov_b32_e32 v5, s23 ; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; HEURRC-NEXT: v_mov_b32_e32 v4, s24 -; HEURRC-NEXT: v_mov_b32_e32 v5, s25 -; HEURRC-NEXT: v_mov_b32_e32 v6, s26 -; HEURRC-NEXT: v_mov_b32_e32 v7, s27 +; HEURRC-NEXT: v_mov_b32_e32 v6, s24 +; HEURRC-NEXT: v_mov_b32_e32 v7, s25 +; HEURRC-NEXT: v_mov_b32_e32 v8, s26 +; HEURRC-NEXT: v_mov_b32_e32 v9, s27 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) ; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23 ; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22 @@ -4408,41 +4400,41 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 ; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9 ; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8 ; HEURRC-NEXT: s_nop 1 -; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] -; HEURRC-NEXT: v_mov_b32_e32 v0, s20 -; HEURRC-NEXT: v_mov_b32_e32 v1, s21 -; HEURRC-NEXT: v_mov_b32_e32 v2, s22 -; HEURRC-NEXT: v_mov_b32_e32 v3, s23 -; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[2:5], v[6:9], a[16:31] +; HEURRC-NEXT: v_mov_b32_e32 v2, s20 +; HEURRC-NEXT: v_mov_b32_e32 v3, s21 +; HEURRC-NEXT: v_mov_b32_e32 v4, s22 +; HEURRC-NEXT: v_mov_b32_e32 v5, s23 +; HEURRC-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:48 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 -; HEURRC-NEXT: v_mov_b32_e32 v0, s16 -; HEURRC-NEXT: v_mov_b32_e32 v1, s17 -; HEURRC-NEXT: v_mov_b32_e32 v2, s18 -; HEURRC-NEXT: v_mov_b32_e32 v3, s19 -; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: v_mov_b32_e32 v2, s16 +; HEURRC-NEXT: v_mov_b32_e32 v3, s17 +; HEURRC-NEXT: v_mov_b32_e32 v4, s18 +; HEURRC-NEXT: v_mov_b32_e32 v5, s19 +; HEURRC-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:32 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 -; HEURRC-NEXT: v_mov_b32_e32 v0, s12 -; HEURRC-NEXT: v_mov_b32_e32 v1, s13 -; HEURRC-NEXT: v_mov_b32_e32 v2, s14 -; HEURRC-NEXT: v_mov_b32_e32 v3, s15 -; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: v_mov_b32_e32 v2, s12 +; HEURRC-NEXT: v_mov_b32_e32 v3, s13 +; HEURRC-NEXT: v_mov_b32_e32 v4, s14 +; HEURRC-NEXT: v_mov_b32_e32 v5, s15 +; HEURRC-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:16 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 -; HEURRC-NEXT: v_mov_b32_e32 v0, s8 -; HEURRC-NEXT: v_mov_b32_e32 v1, s9 -; HEURRC-NEXT: v_mov_b32_e32 v2, s10 -; HEURRC-NEXT: v_mov_b32_e32 v3, s11 -; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: v_mov_b32_e32 v2, s8 +; HEURRC-NEXT: v_mov_b32_e32 v3, s9 +; HEURRC-NEXT: v_mov_b32_e32 v4, s10 +; HEURRC-NEXT: v_mov_b32_e32 v5, s11 +; HEURRC-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_endpgm ; @@ -4450,17 +4442,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 ; VGPRRC: ; %bb.0: ; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 ; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; VGPRRC-NEXT: v_mov_b32_e32 v40, 0 +; VGPRRC-NEXT: v_mov_b32_e32 v32, 0 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) -; VGPRRC-NEXT: v_mov_b32_e32 v32, s20 -; VGPRRC-NEXT: v_mov_b32_e32 v33, s21 -; VGPRRC-NEXT: v_mov_b32_e32 v34, s22 -; VGPRRC-NEXT: v_mov_b32_e32 v35, s23 +; VGPRRC-NEXT: v_mov_b32_e32 v34, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v35, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v36, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v37, s23 ; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; VGPRRC-NEXT: v_mov_b32_e32 v36, s24 -; VGPRRC-NEXT: v_mov_b32_e32 v37, s25 -; VGPRRC-NEXT: v_mov_b32_e32 v38, s26 -; VGPRRC-NEXT: v_mov_b32_e32 v39, s27 +; VGPRRC-NEXT: v_mov_b32_e32 v38, s24 +; VGPRRC-NEXT: v_mov_b32_e32 v39, s25 +; VGPRRC-NEXT: v_mov_b32_e32 v40, s26 +; VGPRRC-NEXT: v_mov_b32_e32 v41, s27 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) ; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23] ; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21] @@ -4471,42 +4463,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 ; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11] ; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9] ; VGPRRC-NEXT: s_nop 1 -; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] ; VGPRRC-NEXT: s_nop 6 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s20 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s21 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s22 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s23 -; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s16 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s17 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s18 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s19 -; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s12 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s13 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s14 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s15 -; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s8 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s9 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s10 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s11 -; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_endpgm ; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd: @@ -4653,17 +4645,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a ; SDAG: ; %bb.0: ; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 ; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; SDAG-NEXT: v_mov_b32_e32 v8, 0 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s20 -; SDAG-NEXT: v_mov_b32_e32 v1, s21 -; SDAG-NEXT: v_mov_b32_e32 v2, s22 -; SDAG-NEXT: v_mov_b32_e32 v3, s23 +; SDAG-NEXT: v_mov_b32_e32 v2, s20 +; SDAG-NEXT: v_mov_b32_e32 v3, s21 +; SDAG-NEXT: v_mov_b32_e32 v4, s22 +; SDAG-NEXT: v_mov_b32_e32 v5, s23 ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; SDAG-NEXT: v_mov_b32_e32 v4, s24 -; SDAG-NEXT: v_mov_b32_e32 v5, s25 -; SDAG-NEXT: v_mov_b32_e32 v6, s26 -; SDAG-NEXT: v_mov_b32_e32 v7, s27 +; SDAG-NEXT: v_mov_b32_e32 v6, s24 +; SDAG-NEXT: v_mov_b32_e32 v7, s25 +; SDAG-NEXT: v_mov_b32_e32 v8, s26 +; SDAG-NEXT: v_mov_b32_e32 v9, s27 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_accvgpr_write_b32 a31, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a30, s22 @@ -4682,41 +4674,41 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a ; SDAG-NEXT: v_accvgpr_write_b32 a17, s9 ; SDAG-NEXT: v_accvgpr_write_b32 a16, s8 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:3 -; SDAG-NEXT: v_mov_b32_e32 v0, s20 -; SDAG-NEXT: v_mov_b32_e32 v1, s21 -; SDAG-NEXT: v_mov_b32_e32 v2, s22 -; SDAG-NEXT: v_mov_b32_e32 v3, s23 -; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc1 +; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[2:5], v[6:9], a[16:31] cbsz:1 abid:2 blgp:3 +; SDAG-NEXT: v_mov_b32_e32 v2, s20 +; SDAG-NEXT: v_mov_b32_e32 v3, s21 +; SDAG-NEXT: v_mov_b32_e32 v4, s22 +; SDAG-NEXT: v_mov_b32_e32 v5, s23 +; SDAG-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:48 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v2, s16 +; SDAG-NEXT: v_mov_b32_e32 v3, s17 +; SDAG-NEXT: v_mov_b32_e32 v4, s18 +; SDAG-NEXT: v_mov_b32_e32 v5, s19 +; SDAG-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:32 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v2, s12 +; SDAG-NEXT: v_mov_b32_e32 v3, s13 +; SDAG-NEXT: v_mov_b32_e32 v4, s14 +; SDAG-NEXT: v_mov_b32_e32 v5, s15 +; SDAG-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:16 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v2, s8 +; SDAG-NEXT: v_mov_b32_e32 v3, s9 +; SDAG-NEXT: v_mov_b32_e32 v4, s10 +; SDAG-NEXT: v_mov_b32_e32 v5, s11 +; SDAG-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; @@ -4778,17 +4770,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a ; HEURRC: ; %bb.0: ; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 ; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, 0 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) -; HEURRC-NEXT: v_mov_b32_e32 v0, s20 -; HEURRC-NEXT: v_mov_b32_e32 v1, s21 -; HEURRC-NEXT: v_mov_b32_e32 v2, s22 -; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: v_mov_b32_e32 v2, s20 +; HEURRC-NEXT: v_mov_b32_e32 v3, s21 +; HEURRC-NEXT: v_mov_b32_e32 v4, s22 +; HEURRC-NEXT: v_mov_b32_e32 v5, s23 ; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; HEURRC-NEXT: v_mov_b32_e32 v4, s24 -; HEURRC-NEXT: v_mov_b32_e32 v5, s25 -; HEURRC-NEXT: v_mov_b32_e32 v6, s26 -; HEURRC-NEXT: v_mov_b32_e32 v7, s27 +; HEURRC-NEXT: v_mov_b32_e32 v6, s24 +; HEURRC-NEXT: v_mov_b32_e32 v7, s25 +; HEURRC-NEXT: v_mov_b32_e32 v8, s26 +; HEURRC-NEXT: v_mov_b32_e32 v9, s27 ; HEURRC-NEXT: s_waitcnt lgkmcnt(0) ; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23 ; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22 @@ -4807,41 +4799,41 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a ; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9 ; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8 ; HEURRC-NEXT: s_nop 1 -; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:3 -; HEURRC-NEXT: v_mov_b32_e32 v0, s20 -; HEURRC-NEXT: v_mov_b32_e32 v1, s21 -; HEURRC-NEXT: v_mov_b32_e32 v2, s22 -; HEURRC-NEXT: v_mov_b32_e32 v3, s23 -; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[2:5], v[6:9], a[16:31] cbsz:1 abid:2 blgp:3 +; HEURRC-NEXT: v_mov_b32_e32 v2, s20 +; HEURRC-NEXT: v_mov_b32_e32 v3, s21 +; HEURRC-NEXT: v_mov_b32_e32 v4, s22 +; HEURRC-NEXT: v_mov_b32_e32 v5, s23 +; HEURRC-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:48 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 -; HEURRC-NEXT: v_mov_b32_e32 v0, s16 -; HEURRC-NEXT: v_mov_b32_e32 v1, s17 -; HEURRC-NEXT: v_mov_b32_e32 v2, s18 -; HEURRC-NEXT: v_mov_b32_e32 v3, s19 -; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: v_mov_b32_e32 v2, s16 +; HEURRC-NEXT: v_mov_b32_e32 v3, s17 +; HEURRC-NEXT: v_mov_b32_e32 v4, s18 +; HEURRC-NEXT: v_mov_b32_e32 v5, s19 +; HEURRC-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:32 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 -; HEURRC-NEXT: v_mov_b32_e32 v0, s12 -; HEURRC-NEXT: v_mov_b32_e32 v1, s13 -; HEURRC-NEXT: v_mov_b32_e32 v2, s14 -; HEURRC-NEXT: v_mov_b32_e32 v3, s15 -; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: v_mov_b32_e32 v2, s12 +; HEURRC-NEXT: v_mov_b32_e32 v3, s13 +; HEURRC-NEXT: v_mov_b32_e32 v4, s14 +; HEURRC-NEXT: v_mov_b32_e32 v5, s15 +; HEURRC-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:16 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_nop 0 -; HEURRC-NEXT: v_mov_b32_e32 v0, s8 -; HEURRC-NEXT: v_mov_b32_e32 v1, s9 -; HEURRC-NEXT: v_mov_b32_e32 v2, s10 -; HEURRC-NEXT: v_mov_b32_e32 v3, s11 -; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: v_mov_b32_e32 v2, s8 +; HEURRC-NEXT: v_mov_b32_e32 v3, s9 +; HEURRC-NEXT: v_mov_b32_e32 v4, s10 +; HEURRC-NEXT: v_mov_b32_e32 v5, s11 +; HEURRC-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) -; HEURRC-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 sc0 sc1 ; HEURRC-NEXT: s_waitcnt vmcnt(0) ; HEURRC-NEXT: s_endpgm ; @@ -4849,17 +4841,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a ; VGPRRC: ; %bb.0: ; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 ; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 -; VGPRRC-NEXT: v_mov_b32_e32 v40, 0 +; VGPRRC-NEXT: v_mov_b32_e32 v32, 0 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) -; VGPRRC-NEXT: v_mov_b32_e32 v32, s20 -; VGPRRC-NEXT: v_mov_b32_e32 v33, s21 -; VGPRRC-NEXT: v_mov_b32_e32 v34, s22 -; VGPRRC-NEXT: v_mov_b32_e32 v35, s23 +; VGPRRC-NEXT: v_mov_b32_e32 v34, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v35, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v36, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v37, s23 ; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 -; VGPRRC-NEXT: v_mov_b32_e32 v36, s24 -; VGPRRC-NEXT: v_mov_b32_e32 v37, s25 -; VGPRRC-NEXT: v_mov_b32_e32 v38, s26 -; VGPRRC-NEXT: v_mov_b32_e32 v39, s27 +; VGPRRC-NEXT: v_mov_b32_e32 v38, s24 +; VGPRRC-NEXT: v_mov_b32_e32 v39, s25 +; VGPRRC-NEXT: v_mov_b32_e32 v40, s26 +; VGPRRC-NEXT: v_mov_b32_e32 v41, s27 ; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) ; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23] ; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21] @@ -4870,42 +4862,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a ; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11] ; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9] ; VGPRRC-NEXT: s_nop 1 -; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3 +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3 ; VGPRRC-NEXT: s_nop 6 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s20 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s21 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s22 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s23 -; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s16 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s17 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s18 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s19 -; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s12 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s13 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s14 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s15 -; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_nop 0 ; VGPRRC-NEXT: v_mov_b32_e32 v16, s8 ; VGPRRC-NEXT: v_mov_b32_e32 v17, s9 ; VGPRRC-NEXT: v_mov_b32_e32 v18, s10 ; VGPRRC-NEXT: v_mov_b32_e32 v19, s11 -; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) -; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1 ; VGPRRC-NEXT: s_waitcnt vmcnt(0) ; VGPRRC-NEXT: s_endpgm ; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll index ccee113..856185b 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll @@ -1,22 +1,116 @@ -; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX908 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX908 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32, i32, <16 x i32>, i32, i32, i32) declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32, i32, <4 x i32>, i32, i32, i32) -; GCN-LABEL: {{^}}test_mfma_i32_32x32x8i8: -; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; GCN-DAG: s_load_dwordx16 -; GFX908-DAG-COUNT-16: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX90A-COUNT-16: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GCN: v_mfma_i32_32x32x8i8 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX908-COUNT-16: v_accvgpr_read_b32 -; GFX908: global_store_dwordx4 -; GFX90A-NOT: v_accvgpr_read_b32 -; GFX90A-COUNT-4: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_i32_32x32x8i8(ptr addrspace(1) %arg) #0 { +; GFX908-LABEL: test_mfma_i32_32x32x8i8: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX908-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX908-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX908-NEXT: s_endpgm +; +; GFX90A-LABEL: test_mfma_i32_32x32x8i8: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v0, 1 +; GFX90A-NEXT: v_mov_b32_e32 v1, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX90A-NEXT: s_endpgm bb: %in.1 = load <16 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32 1, i32 2, <16 x i32> %in.1, i32 1, i32 2, i32 3) @@ -24,18 +118,55 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_i32_16x16x16i8: -; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; GCN: s_load_dwordx4 -; GFX908-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} -; GFX90A-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GCN: v_mfma_i32_16x16x16i8 [[RES:a\[[0-9]+:[0-9]+\]]], [[ONE]], [[TWO]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX908-COUNT-4: v_accvgpr_read_b32 -; GFX908: global_store_dwordx4 -; GFX90A-NOT: v_accvgpr_read_b32 -; GFX90A: global_store_dwordx4 v{{[0-9]+}}, [[RES]] define amdgpu_kernel void @test_mfma_i32_16x16x16i8(ptr addrspace(1) %arg) #0 { +; GFX908-LABEL: test_mfma_i32_16x16x16i8: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v5, s0 +; GFX908-NEXT: v_mov_b32_e32 v2, s1 +; GFX908-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-NEXT: v_mov_b32_e32 v5, s3 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX908-NEXT: s_endpgm +; +; GFX90A-LABEL: test_mfma_i32_16x16x16i8: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v0, 1 +; GFX90A-NEXT: v_mov_b32_e32 v2, 2 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] +; GFX90A-NEXT: s_endpgm bb: %in.1 = load <4 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32 1, i32 2, <4 x i32> %in.1, i32 1, i32 2, i32 3) @@ -44,3 +175,5 @@ bb: } attributes #0 = { "amdgpu-flat-work-group-size"="1,256" } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GCN: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll index ff305da..78be949 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll @@ -3,6 +3,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GCN,LIT-SRCC,GFX908,GFX908_A %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A,GFX908_A,GFX90A_42 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,GFX90A_42 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR %s declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float, float, <16 x float>, i32, i32, i32) @@ -405,6 +406,63 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[34:35] ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[34:35] offset:16 ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v33, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v34, 2.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v32, 0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, s16 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, s17 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, s18 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, s19 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, s20 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, s21 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, s22 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, s23 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, s24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, s25 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, s26 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, s27 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, s28 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, s29 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v14, s30 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v15, s31 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, s0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, s1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, s2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, s3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v20, s4 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v21, s5 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v22, s6 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v23, s7 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v24, s8 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v25, s9 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v26, s10 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v27, s11 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v28, s12 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v29, s13 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v30, s14 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v31, s15 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35] +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16 +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 1, i32 2, i32 3) @@ -618,6 +676,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 { ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, 2.0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> %in.1, i32 1, i32 2, i32 3) @@ -719,6 +804,23 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 { ; GFX942-NEXT: s_nop 3 ; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 2.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 3 +; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> %in.1, i32 1, i32 2, i32 3) @@ -934,6 +1036,34 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 { ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x2f32: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, 2.0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x2_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x2f32(float 1.0, float 2.0, <16 x float> %in.1, i32 1, i32 2, i32 3) @@ -1039,6 +1169,24 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f32(ptr addrspace(1) %arg) #0 { ; GFX942-NEXT: s_nop 1 ; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4f32: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 2.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x4f32(float 1.0, float 2.0, <4 x float> %in.1, i32 1, i32 2, i32 3) @@ -1114,19 +1262,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v2 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v1 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s14 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s15 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s14 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s15 ; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s0 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s1 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s0 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s1 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v6 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v2 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v3 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s2 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s3 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v0 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v1 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s2 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s3 ; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[0:1], v[2:3], a[0:31] cbsz:1 abid:2 blgp:3 +; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[2:3], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3 ; NOLIT-SRCC-NEXT: s_nop 7 ; NOLIT-SRCC-NEXT: s_nop 7 ; NOLIT-SRCC-NEXT: s_nop 1 @@ -1254,19 +1402,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v2 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v1 -; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s14 -; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s15 +; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s14 +; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s15 ; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) -; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s0 -; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s1 +; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s0 +; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s1 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v6 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v2 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v3 -; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s2 -; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s3 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v0 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v1 +; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s2 +; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s3 ; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[0:1], v[2:3], a[0:31] cbsz:1 abid:2 blgp:3 +; LIT-SRCC-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[2:3], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3 ; LIT-SRCC-NEXT: s_nop 7 ; LIT-SRCC-NEXT: s_nop 7 ; LIT-SRCC-NEXT: s_nop 1 @@ -1330,7 +1478,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; GFX90A-LABEL: test_mfma_f32_32x32x4f16: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x40 ; GFX90A-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x0 @@ -1345,8 +1493,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; GFX90A-NEXT: v_accvgpr_write_b32 a2, s18 ; GFX90A-NEXT: v_accvgpr_write_b32 a3, s19 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v0, s0 -; GFX90A-NEXT: v_mov_b32_e32 v1, s1 +; GFX90A-NEXT: v_mov_b32_e32 v2, s0 +; GFX90A-NEXT: v_mov_b32_e32 v3, s1 ; GFX90A-NEXT: v_accvgpr_write_b32 a4, s20 ; GFX90A-NEXT: v_accvgpr_write_b32 a5, s21 ; GFX90A-NEXT: v_accvgpr_write_b32 a6, s22 @@ -1371,27 +1519,27 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; GFX90A-NEXT: v_accvgpr_write_b32 a29, s13 ; GFX90A-NEXT: v_accvgpr_write_b32 a30, s14 ; GFX90A-NEXT: v_accvgpr_write_b32 a31, s15 -; GFX90A-NEXT: v_mov_b32_e32 v2, s2 -; GFX90A-NEXT: v_mov_b32_e32 v3, s3 +; GFX90A-NEXT: v_mov_b32_e32 v4, s2 +; GFX90A-NEXT: v_mov_b32_e32 v5, s3 ; GFX90A-NEXT: s_nop 1 -; GFX90A-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[0:1], v[2:3], a[0:31] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[2:3], v[4:5], a[0:31] cbsz:1 abid:2 blgp:3 ; GFX90A-NEXT: s_nop 7 ; GFX90A-NEXT: s_nop 7 ; GFX90A-NEXT: s_nop 2 -; GFX90A-NEXT: global_store_dwordx4 v4, a[24:27], s[36:37] offset:96 -; GFX90A-NEXT: global_store_dwordx4 v4, a[28:31], s[36:37] offset:112 -; GFX90A-NEXT: global_store_dwordx4 v4, a[16:19], s[36:37] offset:64 -; GFX90A-NEXT: global_store_dwordx4 v4, a[20:23], s[36:37] offset:80 -; GFX90A-NEXT: global_store_dwordx4 v4, a[8:11], s[36:37] offset:32 -; GFX90A-NEXT: global_store_dwordx4 v4, a[12:15], s[36:37] offset:48 -; GFX90A-NEXT: global_store_dwordx4 v4, a[0:3], s[36:37] -; GFX90A-NEXT: global_store_dwordx4 v4, a[4:7], s[36:37] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[24:27], s[36:37] offset:96 +; GFX90A-NEXT: global_store_dwordx4 v0, a[28:31], s[36:37] offset:112 +; GFX90A-NEXT: global_store_dwordx4 v0, a[16:19], s[36:37] offset:64 +; GFX90A-NEXT: global_store_dwordx4 v0, a[20:23], s[36:37] offset:80 +; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[36:37] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[36:37] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[36:37] +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[36:37] offset:16 ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: test_mfma_f32_32x32x4f16: ; GFX942: ; %bb.0: ; %bb ; GFX942-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24 -; GFX942-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x40 ; GFX942-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x0 @@ -1406,8 +1554,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; GFX942-NEXT: v_accvgpr_write_b32 a2, s18 ; GFX942-NEXT: v_accvgpr_write_b32 a3, s19 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b32_e32 v0, s0 -; GFX942-NEXT: v_mov_b32_e32 v1, s1 +; GFX942-NEXT: v_mov_b32_e32 v2, s0 +; GFX942-NEXT: v_mov_b32_e32 v3, s1 ; GFX942-NEXT: v_accvgpr_write_b32 a4, s20 ; GFX942-NEXT: v_accvgpr_write_b32 a5, s21 ; GFX942-NEXT: v_accvgpr_write_b32 a6, s22 @@ -1432,22 +1580,83 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; GFX942-NEXT: v_accvgpr_write_b32 a29, s13 ; GFX942-NEXT: v_accvgpr_write_b32 a30, s14 ; GFX942-NEXT: v_accvgpr_write_b32 a31, s15 -; GFX942-NEXT: v_mov_b32_e32 v2, s2 -; GFX942-NEXT: v_mov_b32_e32 v3, s3 +; GFX942-NEXT: v_mov_b32_e32 v4, s2 +; GFX942-NEXT: v_mov_b32_e32 v5, s3 ; GFX942-NEXT: s_nop 1 -; GFX942-NEXT: v_mfma_f32_32x32x4_2b_f16 a[0:31], v[0:1], v[2:3], a[0:31] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: v_mfma_f32_32x32x4_2b_f16 a[0:31], v[2:3], v[4:5], a[0:31] cbsz:1 abid:2 blgp:3 ; GFX942-NEXT: s_nop 7 ; GFX942-NEXT: s_nop 7 ; GFX942-NEXT: s_nop 2 -; GFX942-NEXT: global_store_dwordx4 v4, a[24:27], s[36:37] offset:96 -; GFX942-NEXT: global_store_dwordx4 v4, a[28:31], s[36:37] offset:112 -; GFX942-NEXT: global_store_dwordx4 v4, a[16:19], s[36:37] offset:64 -; GFX942-NEXT: global_store_dwordx4 v4, a[20:23], s[36:37] offset:80 -; GFX942-NEXT: global_store_dwordx4 v4, a[8:11], s[36:37] offset:32 -; GFX942-NEXT: global_store_dwordx4 v4, a[12:15], s[36:37] offset:48 -; GFX942-NEXT: global_store_dwordx4 v4, a[0:3], s[36:37] -; GFX942-NEXT: global_store_dwordx4 v4, a[4:7], s[36:37] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[24:27], s[36:37] offset:96 +; GFX942-NEXT: global_store_dwordx4 v0, a[28:31], s[36:37] offset:112 +; GFX942-NEXT: global_store_dwordx4 v0, a[16:19], s[36:37] offset:64 +; GFX942-NEXT: global_store_dwordx4 v0, a[20:23], s[36:37] offset:80 +; GFX942-NEXT: global_store_dwordx4 v0, a[8:11], s[36:37] offset:32 +; GFX942-NEXT: global_store_dwordx4 v0, a[12:15], s[36:37] offset:48 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[36:37] +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[36:37] offset:16 ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x4f16: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v32, 0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x40 +; GFX942-VGPR-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, s0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, s1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, s2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, s3 +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[38:39], 0x0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, s16 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, s17 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, s18 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, s19 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v34, s0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v35, s1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, s20 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, s21 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, s22 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, s23 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, s24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, s25 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, s26 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, s27 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, s28 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, s29 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v14, s30 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v15, s31 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v20, s4 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v21, s5 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v22, s6 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v23, s7 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v24, s8 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v25, s9 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v26, s10 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v27, s11 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v28, s12 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v29, s13 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v30, s14 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v31, s15 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v36, s2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v37, s3 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x4_2b_f16 v[0:31], v[34:35], v[36:37], v[0:31] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37] +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16 +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg %c.1 = load <4 x half>, ptr addrspace(1) %c @@ -1676,6 +1885,36 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4f16: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0 +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, s20 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, s21 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, s22 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, s23 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x4_4b_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %c.1 = load <4 x half>, ptr addrspace(1) %c @@ -1752,46 +1991,66 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add ; GFX90A-LABEL: test_mfma_f32_4x4x4f16: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 ; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v0, s4 -; GFX90A-NEXT: v_mov_b32_e32 v1, s5 +; GFX90A-NEXT: v_mov_b32_e32 v2, s4 +; GFX90A-NEXT: v_mov_b32_e32 v3, s5 ; GFX90A-NEXT: v_accvgpr_write_b32 a0, s8 -; GFX90A-NEXT: v_mov_b32_e32 v2, s6 -; GFX90A-NEXT: v_mov_b32_e32 v3, s7 +; GFX90A-NEXT: v_mov_b32_e32 v4, s6 +; GFX90A-NEXT: v_mov_b32_e32 v5, s7 ; GFX90A-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX90A-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX90A-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX90A-NEXT: s_nop 1 -; GFX90A-NEXT: v_mfma_f32_4x4x4f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mfma_f32_4x4x4f16 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX90A-NEXT: s_nop 4 -; GFX90A-NEXT: global_store_dwordx4 v4, a[0:3], s[0:1] +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: test_mfma_f32_4x4x4f16: ; GFX942: ; %bb.0: ; %bb ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 -; GFX942-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 ; GFX942-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b32_e32 v0, s4 -; GFX942-NEXT: v_mov_b32_e32 v1, s5 +; GFX942-NEXT: v_mov_b32_e32 v2, s4 +; GFX942-NEXT: v_mov_b32_e32 v3, s5 ; GFX942-NEXT: v_accvgpr_write_b32 a0, s8 -; GFX942-NEXT: v_mov_b32_e32 v2, s6 -; GFX942-NEXT: v_mov_b32_e32 v3, s7 +; GFX942-NEXT: v_mov_b32_e32 v4, s6 +; GFX942-NEXT: v_mov_b32_e32 v5, s7 ; GFX942-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-NEXT: s_nop 1 -; GFX942-NEXT: v_mfma_f32_4x4x4_16b_f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: v_mfma_f32_4x4x4_16b_f16 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-NEXT: s_nop 4 -; GFX942-NEXT: global_store_dwordx4 v4, a[0:3], s[0:1] +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_4x4x4f16: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 +; GFX942-VGPR-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, s4 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, s5 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, s6 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, s7 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_4x4x4_16b_f16 v[0:3], v[6:7], v[8:9], v[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 4 +; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %c.1 = load <4 x half>, ptr addrspace(1) %c @@ -2021,6 +2280,36 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8f16: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0 +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, s20 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, s21 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, s22 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, s23 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %c.1 = load <4 x half>, ptr addrspace(1) %c @@ -2099,47 +2388,67 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr ; GFX90A-LABEL: test_mfma_f32_16x16x16f16: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 ; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v0, s4 -; GFX90A-NEXT: v_mov_b32_e32 v1, s5 +; GFX90A-NEXT: v_mov_b32_e32 v2, s4 +; GFX90A-NEXT: v_mov_b32_e32 v3, s5 ; GFX90A-NEXT: v_accvgpr_write_b32 a0, s8 -; GFX90A-NEXT: v_mov_b32_e32 v2, s6 -; GFX90A-NEXT: v_mov_b32_e32 v3, s7 +; GFX90A-NEXT: v_mov_b32_e32 v4, s6 +; GFX90A-NEXT: v_mov_b32_e32 v5, s7 ; GFX90A-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX90A-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX90A-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX90A-NEXT: s_nop 1 -; GFX90A-NEXT: v_mfma_f32_16x16x16f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mfma_f32_16x16x16f16 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX90A-NEXT: s_nop 7 ; GFX90A-NEXT: s_nop 2 -; GFX90A-NEXT: global_store_dwordx4 v4, a[0:3], s[0:1] +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX90A-NEXT: s_endpgm ; ; GFX942-LABEL: test_mfma_f32_16x16x16f16: ; GFX942: ; %bb.0: ; %bb ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 -; GFX942-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 ; GFX942-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b32_e32 v0, s4 -; GFX942-NEXT: v_mov_b32_e32 v1, s5 +; GFX942-NEXT: v_mov_b32_e32 v2, s4 +; GFX942-NEXT: v_mov_b32_e32 v3, s5 ; GFX942-NEXT: v_accvgpr_write_b32 a0, s8 -; GFX942-NEXT: v_mov_b32_e32 v2, s6 -; GFX942-NEXT: v_mov_b32_e32 v3, s7 +; GFX942-NEXT: v_mov_b32_e32 v4, s6 +; GFX942-NEXT: v_mov_b32_e32 v5, s7 ; GFX942-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-NEXT: s_nop 1 -; GFX942-NEXT: v_mfma_f32_16x16x16_f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: v_mfma_f32_16x16x16_f16 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-NEXT: s_nop 6 -; GFX942-NEXT: global_store_dwordx4 v4, a[0:3], s[0:1] +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_16x16x16f16: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 +; GFX942-VGPR-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, s4 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, s5 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, s6 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, s7 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x16_f16 v[0:3], v[6:7], v[8:9], v[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 6 +; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %c.1 = load <4 x half>, ptr addrspace(1) %c @@ -2508,6 +2817,63 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[34:35] ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[34:35] offset:16 ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_i32_32x32x4i8: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v33, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v34, 2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v32, 0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, s16 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, s17 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, s18 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, s19 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, s20 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, s21 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, s22 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, s23 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, s24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, s25 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, s26 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, s27 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, s28 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, s29 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v14, s30 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v15, s31 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, s0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, s1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, s2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, s3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v20, s4 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v21, s5 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v22, s6 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v23, s7 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v24, s8 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v25, s9 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v26, s10 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v27, s11 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v28, s12 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v29, s13 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v30, s14 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v31, s15 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_i32_32x32x4_2b_i8 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35] +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16 +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <32 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <32 x i32> @llvm.amdgcn.mfma.i32.32x32x4i8(i32 1, i32 2, <32 x i32> %in.1, i32 1, i32 2, i32 3) @@ -2721,6 +3087,33 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 { ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_i32_16x16x4i8: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, 2 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.16x16x4i8(i32 1, i32 2, <16 x i32> %in.1, i32 1, i32 2, i32 3) @@ -2728,6 +3121,143 @@ bb: ret void } +define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspace(1) %arg) #0 { +; NOLIT-SRCC-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64: +; NOLIT-SRCC: ; %bb.0: ; %bb +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, 64 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, 64 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2 +; NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0 +; NOLIT-SRCC-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; NOLIT-SRCC-NEXT: s_nop 7 +; NOLIT-SRCC-NEXT: s_nop 1 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8 +; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; NOLIT-SRCC-NEXT: s_endpgm +; +; LIT-SRCC-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64: +; LIT-SRCC: ; %bb.0: ; %bb +; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1 +; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2 +; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0 +; LIT-SRCC-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, 64 cbsz:1 abid:2 blgp:3 +; LIT-SRCC-NEXT: s_nop 7 +; LIT-SRCC-NEXT: s_nop 1 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8 +; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; LIT-SRCC-NEXT: s_endpgm +; +; GFX90A-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: v_mov_b32_e32 v0, 1 +; GFX90A-NEXT: v_mov_b32_e32 v1, 2 +; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, 64 cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: v_mov_b32_e32 v0, 1 +; GFX942-NEXT: v_mov_b32_e32 v1, 2 +; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: v_mfma_i32_16x16x4_4b_i8 a[0:15], v0, v1, 64 cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 2 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPR-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v0, v1, 64 cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm +bb: + %in.1 = load <16 x i32>, ptr addrspace(1) %arg + %mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.16x16x4i8(i32 1, i32 2, <16 x i32> splat (i32 64), i32 1, i32 2, i32 3) + store <16 x i32> %mai.1, ptr addrspace(1) %arg + ret void +} + define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-LABEL: test_mfma_i32_4x4x4i8: ; NOLIT-SRCC: ; %bb.0: ; %bb @@ -2822,6 +3352,23 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 { ; GFX942-NEXT: s_nop 4 ; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 4 +; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> %in.1, i32 1, i32 2, i32 3) @@ -2829,6 +3376,197 @@ bb: ret void } +define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(1) %arg) #0 { +; NOLIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1: +; NOLIT-SRCC: ; %bb.0: ; %bb +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 1 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 1 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 1 +; NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; NOLIT-SRCC-NEXT: s_nop 0 +; NOLIT-SRCC-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3 +; NOLIT-SRCC-NEXT: s_nop 3 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; NOLIT-SRCC-NEXT: s_nop 0 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; NOLIT-SRCC-NEXT: s_endpgm +; +; LIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1: +; LIT-SRCC: ; %bb.0: ; %bb +; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1 +; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2 +; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; LIT-SRCC-NEXT: s_nop 0 +; LIT-SRCC-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, 1 cbsz:1 abid:2 blgp:3 +; LIT-SRCC-NEXT: s_nop 3 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; LIT-SRCC-NEXT: s_nop 0 +; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; LIT-SRCC-NEXT: s_endpgm +; +; GFX90A-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: v_mov_b32_e32 v0, 1 +; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v2, 2 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v2, 1 cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_nop 3 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: v_mov_b32_e32 v0, 1 +; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: v_mfma_i32_4x4x4_16b_i8 a[0:3], v0, v2, 1 cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_nop 3 +; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v0, v1, 1 cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 3 +; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm +bb: + %in.1 = load <4 x i32>, ptr addrspace(1) %arg + %mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> splat (i32 1), i32 1, i32 2, i32 3) + store <4 x i32> %mai.1, ptr addrspace(1) %arg + ret void +} + +define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1) %arg) #0 { +; NOLIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1: +; NOLIT-SRCC: ; %bb.0: +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 0x41 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 1 +; NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v0 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v0 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v0 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 2 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; NOLIT-SRCC-NEXT: s_nop 0 +; NOLIT-SRCC-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v1, v0, a[0:3] cbsz:1 abid:2 blgp:3 +; NOLIT-SRCC-NEXT: s_nop 3 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; NOLIT-SRCC-NEXT: s_nop 0 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; NOLIT-SRCC-NEXT: s_endpgm +; +; LIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1: +; LIT-SRCC: ; %bb.0: +; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 0x41 +; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 1 +; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v0 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v0 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v0 +; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 2 +; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; LIT-SRCC-NEXT: s_nop 0 +; LIT-SRCC-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v1, v0, a[0:3] cbsz:1 abid:2 blgp:3 +; LIT-SRCC-NEXT: s_nop 3 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; LIT-SRCC-NEXT: s_nop 0 +; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; LIT-SRCC-NEXT: s_endpgm +; +; GFX90A-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1: +; GFX90A: ; %bb.0: +; GFX90A-NEXT: v_mov_b32_e32 v1, 0x41 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, v1 +; GFX90A-NEXT: v_mov_b32_e32 v1, 1 +; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX90A-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX90A-NEXT: v_mov_b32_e32 v2, 2 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v1, v2, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_nop 3 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1: +; GFX942: ; %bb.0: +; GFX942-NEXT: v_mov_b32_e32 v1, 0x41 +; GFX942-NEXT: v_accvgpr_write_b32 a0, v1 +; GFX942-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX942-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: v_mfma_i32_4x4x4_16b_i8 a[0:3], v1, v2, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_nop 3 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1: +; GFX942-VGPR: ; %bb.0: +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 0x41 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v5, v6, v[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 3 +; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm + %in.1 = load <4 x i32>, ptr addrspace(1) %arg + %mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> splat (i32 65), i32 1, i32 2, i32 3) + store <4 x i32> %mai.1, ptr addrspace(1) %arg + ret void +} + define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_forward_acc: ; NOLIT-SRCC: ; %bb.0: ; %bb @@ -3219,6 +3957,64 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1) ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[34:35] ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[34:35] offset:16 ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_forward_acc: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v32, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v33, 2.0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, s16 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, s17 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, s18 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, s19 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, s20 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, s21 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, s22 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, s23 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, s24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, s25 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, s26 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, s27 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, s28 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, s29 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v14, s30 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v15, s31 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, s0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, s1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, s2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, s3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v20, s4 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v21, s5 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v22, s6 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v23, s7 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v24, s8 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v25, s9 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v26, s10 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v27, s11 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v28, s12 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v29, s13 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v30, s14 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v31, s15 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v32, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35] +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16 +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0) @@ -3435,6 +4231,34 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1) ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_forward_acc: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, 2.0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15] +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> %in.1, i32 0, i32 0, i32 0) @@ -3542,6 +4366,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) % ; GFX942-NEXT: s_nop 3 ; GFX942-NEXT: global_store_dwordx4 v2, a[0:3], s[6:7] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_forward_acc: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 2.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3] +; GFX942-VGPR-NEXT: s_nop 3 +; GFX942-VGPR-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> %in.1, i32 0, i32 0, i32 0) @@ -3616,6 +4459,19 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar ; GFX942-NEXT: s_nop 2 ; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[0:1] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_imm_splat: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v0, v1, 1.0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0) store <4 x float> %mai.1, ptr addrspace(1) %arg @@ -3745,6 +4601,22 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) % ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_imm_splat: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v0, v1, 1.0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0) store <16 x float> %mai.1, ptr addrspace(1) %arg @@ -3885,6 +4757,24 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) % ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8f16_imm_splat: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 0x3c003c00 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, 0x40004000 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v2 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[0:1], v[2:3], 1.0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> <half 1.0, half 1.0, half 1.0, half 1.0>, <4 x half> <half 2.0, half 2.0, half 2.0, half 2.0>, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0) store <16 x float> %mai.1, ptr addrspace(1) %arg @@ -4091,6 +4981,27 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) % ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_imm_splat: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v32, 0 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v0, v1, 0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, i32 0, i32 0, i32 0) store <32 x float> %mai.1, ptr addrspace(1) %arg @@ -4175,6 +5086,21 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0 ; GFX942-NEXT: s_nop 2 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_imm: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v0, v1, v[0:3] +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> <float 1.0, float 2.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0) store <4 x float> %mai.1, ptr addrspace(1) %arg @@ -4355,6 +5281,36 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) # ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_imm: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v15, 2.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v14, v0 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v0, v15, v[0:15] +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 2.0>, i32 0, i32 0, i32 0) store <16 x float> %mai.1, ptr addrspace(1) %arg @@ -4667,6 +5623,74 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) # ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_imm: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v14, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v15, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v20, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v21, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v22, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v23, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v24, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v25, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v26, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v27, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v28, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v29, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v30, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v31, v1 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[32:33], v[30:31] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v34, 2.0 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[30:31], v[28:29] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[28:29], v[26:27] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[26:27], v[24:25] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[24:25], v[22:23] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[22:23], v[20:21] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[20:21], v[18:19] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[18:19], v[16:17] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[16:17], v[14:15] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[14:15], v[12:13] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], v[10:11] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], v[8:9] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], v[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], v[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], v[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], v[0:1] +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x1_2b_f32 v[2:33], v0, v34, v[2:33] +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[30:33], s[0:1] offset:112 +; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[26:29], s[0:1] offset:96 +; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[22:25], s[0:1] offset:80 +; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[18:21], s[0:1] offset:64 +; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[14:17], s[0:1] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[10:13], s[0:1] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[6:9], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[2:5], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> <float 1.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, i32 0, i32 0, i32 0) store <32 x float> %mai.1, ptr addrspace(1) %arg @@ -4755,6 +5779,24 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar ; GFX942-NEXT: s_nop 2 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_lit_splat: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1.0 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX942-VGPR-NEXT: v_lshlrev_b32_e32 v4, 4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 0x42f60000 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 2.0 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v5, v6, v[0:3] +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %tid = call i32 @llvm.amdgcn.workitem.id.x() %gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %arg, i32 %tid @@ -4846,6 +5888,23 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa ; GFX942-NEXT: s_nop 2 ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1.0 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 0x42f60000 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 2.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v5, v6, v[0:3] +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %tid = call i32 @llvm.amdgcn.workitem.id.x() %gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %arg, i32 %tid @@ -5109,6 +6168,37 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg ; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_vecarg: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX942-VGPR-NEXT: v_lshlrev_b32_e32 v32, 7, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v33, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v34, 2.0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112 +; GFX942-VGPR-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96 +; GFX942-VGPR-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80 +; GFX942-VGPR-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64 +; GFX942-VGPR-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48 +; GFX942-VGPR-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32 +; GFX942-VGPR-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1] +; GFX942-VGPR-NEXT: s_waitcnt vmcnt(0) +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: s_endpgm bb: %tid = call i32 @llvm.amdgcn.workitem.id.x() %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll index 04ee0bb..37809da 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll @@ -1485,30 +1485,30 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgprs(<8 x i32> inr ; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgprs: ; SDAG: ; %bb.0: ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v12, s0 -; SDAG-NEXT: v_mov_b32_e32 v13, s1 -; SDAG-NEXT: v_mov_b32_e32 v14, s2 -; SDAG-NEXT: v_mov_b32_e32 v15, s3 -; SDAG-NEXT: v_mov_b32_e32 v16, s16 -; SDAG-NEXT: v_mov_b32_e32 v17, s17 -; SDAG-NEXT: v_mov_b32_e32 v18, s18 -; SDAG-NEXT: v_mov_b32_e32 v19, s19 -; SDAG-NEXT: v_mov_b32_e32 v20, s28 -; SDAG-NEXT: v_mov_b32_e32 v21, s29 -; SDAG-NEXT: v_mov_b32_e32 v4, s20 -; SDAG-NEXT: v_mov_b32_e32 v5, s21 -; SDAG-NEXT: v_mov_b32_e32 v6, s22 -; SDAG-NEXT: v_mov_b32_e32 v7, s23 -; SDAG-NEXT: v_mov_b32_e32 v8, s24 -; SDAG-NEXT: v_mov_b32_e32 v9, s25 -; SDAG-NEXT: v_mov_b32_e32 v10, s26 -; SDAG-NEXT: v_mov_b32_e32 v11, s27 -; SDAG-NEXT: v_accvgpr_write_b32 a0, v20 -; SDAG-NEXT: v_accvgpr_write_b32 a1, v21 +; SDAG-NEXT: v_mov_b32_e32 v14, s0 +; SDAG-NEXT: v_mov_b32_e32 v15, s1 +; SDAG-NEXT: v_mov_b32_e32 v16, s2 +; SDAG-NEXT: v_mov_b32_e32 v17, s3 +; SDAG-NEXT: v_mov_b32_e32 v18, s16 +; SDAG-NEXT: v_mov_b32_e32 v19, s17 +; SDAG-NEXT: v_mov_b32_e32 v20, s18 +; SDAG-NEXT: v_mov_b32_e32 v21, s19 +; SDAG-NEXT: v_mov_b32_e32 v4, s28 +; SDAG-NEXT: v_mov_b32_e32 v5, s29 +; SDAG-NEXT: v_mov_b32_e32 v6, s20 +; SDAG-NEXT: v_mov_b32_e32 v7, s21 +; SDAG-NEXT: v_mov_b32_e32 v8, s22 +; SDAG-NEXT: v_mov_b32_e32 v9, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s24 +; SDAG-NEXT: v_mov_b32_e32 v11, s25 +; SDAG-NEXT: v_mov_b32_e32 v12, s26 +; SDAG-NEXT: v_mov_b32_e32 v13, s27 +; SDAG-NEXT: v_accvgpr_write_b32 a0, v4 +; SDAG-NEXT: v_accvgpr_write_b32 a1, v5 ; SDAG-NEXT: v_accvgpr_write_b32 a2, v0 ; SDAG-NEXT: v_accvgpr_write_b32 a3, v1 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[12:19], v[4:11], a[0:3], v2, v3 op_sel_hi:[0,0,0] +; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[14:21], v[6:13], a[0:3], v2, v3 op_sel_hi:[0,0,0] ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 3 ; SDAG-NEXT: v_accvgpr_read_b32 v0, a0 @@ -1895,36 +1895,36 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32 ; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd: ; SDAG: ; %bb.0: ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: v_mov_b32_e32 v4, s12 -; SDAG-NEXT: v_mov_b32_e32 v5, s13 -; SDAG-NEXT: v_mov_b32_e32 v6, s14 -; SDAG-NEXT: v_mov_b32_e32 v7, s15 +; SDAG-NEXT: v_mov_b32_e32 v2, s8 +; SDAG-NEXT: v_mov_b32_e32 v3, s9 +; SDAG-NEXT: v_mov_b32_e32 v4, s10 +; SDAG-NEXT: v_mov_b32_e32 v5, s11 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 ; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40 -; SDAG-NEXT: v_mov_b32_e32 v8, s16 -; SDAG-NEXT: v_mov_b32_e32 v9, s17 -; SDAG-NEXT: v_mov_b32_e32 v10, s18 -; SDAG-NEXT: v_mov_b32_e32 v11, s19 -; SDAG-NEXT: v_mov_b32_e32 v12, s20 -; SDAG-NEXT: v_mov_b32_e32 v13, s21 -; SDAG-NEXT: v_mov_b32_e32 v14, s22 -; SDAG-NEXT: v_mov_b32_e32 v15, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s16 +; SDAG-NEXT: v_mov_b32_e32 v11, s17 +; SDAG-NEXT: v_mov_b32_e32 v12, s18 +; SDAG-NEXT: v_mov_b32_e32 v13, s19 +; SDAG-NEXT: v_mov_b32_e32 v14, s20 +; SDAG-NEXT: v_mov_b32_e32 v15, s21 +; SDAG-NEXT: v_mov_b32_e32 v16, s22 +; SDAG-NEXT: v_mov_b32_e32 v17, s23 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s11 -; SDAG-NEXT: v_mov_b32_e32 v17, s13 +; SDAG-NEXT: v_mov_b32_e32 v1, s13 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], s12, v17 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2 +; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[2:9], v[10:17], a[0:3], s12, v1 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 3 -; SDAG-NEXT: global_store_dwordx4 v16, a[0:3], s[14:15] +; SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[14:15] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd: @@ -1964,33 +1964,33 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40 ; SDAG-NEXT: s_movk_i32 s6, 0x41 ; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: v_mov_b32_e32 v4, s12 -; SDAG-NEXT: v_mov_b32_e32 v5, s13 -; SDAG-NEXT: v_mov_b32_e32 v6, s14 -; SDAG-NEXT: v_mov_b32_e32 v7, s15 +; SDAG-NEXT: v_mov_b32_e32 v2, s8 +; SDAG-NEXT: v_mov_b32_e32 v3, s9 +; SDAG-NEXT: v_mov_b32_e32 v4, s10 +; SDAG-NEXT: v_mov_b32_e32 v5, s11 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s0 -; SDAG-NEXT: v_mov_b32_e32 v8, s16 -; SDAG-NEXT: v_mov_b32_e32 v9, s17 -; SDAG-NEXT: v_mov_b32_e32 v10, s18 -; SDAG-NEXT: v_mov_b32_e32 v11, s19 -; SDAG-NEXT: v_mov_b32_e32 v12, s20 -; SDAG-NEXT: v_mov_b32_e32 v13, s21 -; SDAG-NEXT: v_mov_b32_e32 v14, s22 -; SDAG-NEXT: v_mov_b32_e32 v15, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s16 +; SDAG-NEXT: v_mov_b32_e32 v11, s17 +; SDAG-NEXT: v_mov_b32_e32 v12, s18 +; SDAG-NEXT: v_mov_b32_e32 v13, s19 +; SDAG-NEXT: v_mov_b32_e32 v14, s20 +; SDAG-NEXT: v_mov_b32_e32 v15, s21 +; SDAG-NEXT: v_mov_b32_e32 v16, s22 +; SDAG-NEXT: v_mov_b32_e32 v17, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], s6, -2 op_sel:[1,1,0] op_sel_hi:[1,0,0] +; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[2:9], v[10:17], a[0:3], s6, -2 op_sel:[1,1,0] op_sel_hi:[1,0,0] ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 3 -; SDAG-NEXT: global_store_dwordx4 v16, a[0:3], s[4:5] +; SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[4:5] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm: @@ -2031,33 +2031,33 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40 ; SDAG-NEXT: s_movk_i32 s6, 0x41 ; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: v_mov_b32_e32 v4, s12 -; SDAG-NEXT: v_mov_b32_e32 v5, s13 -; SDAG-NEXT: v_mov_b32_e32 v6, s14 -; SDAG-NEXT: v_mov_b32_e32 v7, s15 +; SDAG-NEXT: v_mov_b32_e32 v2, s8 +; SDAG-NEXT: v_mov_b32_e32 v3, s9 +; SDAG-NEXT: v_mov_b32_e32 v4, s10 +; SDAG-NEXT: v_mov_b32_e32 v5, s11 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s0 -; SDAG-NEXT: v_mov_b32_e32 v8, s16 -; SDAG-NEXT: v_mov_b32_e32 v9, s17 -; SDAG-NEXT: v_mov_b32_e32 v10, s18 -; SDAG-NEXT: v_mov_b32_e32 v11, s19 -; SDAG-NEXT: v_mov_b32_e32 v12, s20 -; SDAG-NEXT: v_mov_b32_e32 v13, s21 -; SDAG-NEXT: v_mov_b32_e32 v14, s22 -; SDAG-NEXT: v_mov_b32_e32 v15, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s16 +; SDAG-NEXT: v_mov_b32_e32 v11, s17 +; SDAG-NEXT: v_mov_b32_e32 v12, s18 +; SDAG-NEXT: v_mov_b32_e32 v13, s19 +; SDAG-NEXT: v_mov_b32_e32 v14, s20 +; SDAG-NEXT: v_mov_b32_e32 v15, s21 +; SDAG-NEXT: v_mov_b32_e32 v16, s22 +; SDAG-NEXT: v_mov_b32_e32 v17, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], s6, 1.0 op_sel:[1,1,0] op_sel_hi:[1,0,0] +; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[2:9], v[10:17], a[0:3], s6, 1.0 op_sel:[1,1,0] op_sel_hi:[1,0,0] ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 3 -; SDAG-NEXT: global_store_dwordx4 v16, a[0:3], s[4:5] +; SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[4:5] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal: @@ -2096,34 +2096,34 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA ; SDAG: ; %bb.0: ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: v_mov_b32_e32 v4, s12 -; SDAG-NEXT: v_mov_b32_e32 v5, s13 -; SDAG-NEXT: v_mov_b32_e32 v6, s14 -; SDAG-NEXT: v_mov_b32_e32 v7, s15 +; SDAG-NEXT: v_mov_b32_e32 v2, s8 +; SDAG-NEXT: v_mov_b32_e32 v3, s9 +; SDAG-NEXT: v_mov_b32_e32 v4, s10 +; SDAG-NEXT: v_mov_b32_e32 v5, s11 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s0 -; SDAG-NEXT: v_mov_b32_e32 v8, s16 -; SDAG-NEXT: v_mov_b32_e32 v9, s17 -; SDAG-NEXT: v_mov_b32_e32 v10, s18 -; SDAG-NEXT: v_mov_b32_e32 v11, s19 -; SDAG-NEXT: v_mov_b32_e32 v12, s20 -; SDAG-NEXT: v_mov_b32_e32 v13, s21 -; SDAG-NEXT: v_mov_b32_e32 v14, s22 -; SDAG-NEXT: v_mov_b32_e32 v15, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s16 +; SDAG-NEXT: v_mov_b32_e32 v11, s17 +; SDAG-NEXT: v_mov_b32_e32 v12, s18 +; SDAG-NEXT: v_mov_b32_e32 v13, s19 +; SDAG-NEXT: v_mov_b32_e32 v14, s20 +; SDAG-NEXT: v_mov_b32_e32 v15, s21 +; SDAG-NEXT: v_mov_b32_e32 v16, s22 +; SDAG-NEXT: v_mov_b32_e32 v17, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], 1.0, -2 op_sel:[1,1,0] op_sel_hi:[1,0,0] +; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[2:9], v[10:17], a[0:3], 1.0, -2 op_sel:[1,1,0] op_sel_hi:[1,0,0] ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 3 -; SDAG-NEXT: global_store_dwordx4 v16, a[0:3], s[4:5] +; SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[4:5] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm: @@ -2162,34 +2162,34 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA ; SDAG: ; %bb.0: ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: v_mov_b32_e32 v4, s12 -; SDAG-NEXT: v_mov_b32_e32 v5, s13 -; SDAG-NEXT: v_mov_b32_e32 v6, s14 -; SDAG-NEXT: v_mov_b32_e32 v7, s15 +; SDAG-NEXT: v_mov_b32_e32 v2, s8 +; SDAG-NEXT: v_mov_b32_e32 v3, s9 +; SDAG-NEXT: v_mov_b32_e32 v4, s10 +; SDAG-NEXT: v_mov_b32_e32 v5, s11 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s0 -; SDAG-NEXT: v_mov_b32_e32 v8, s16 -; SDAG-NEXT: v_mov_b32_e32 v9, s17 -; SDAG-NEXT: v_mov_b32_e32 v10, s18 -; SDAG-NEXT: v_mov_b32_e32 v11, s19 -; SDAG-NEXT: v_mov_b32_e32 v12, s20 -; SDAG-NEXT: v_mov_b32_e32 v13, s21 -; SDAG-NEXT: v_mov_b32_e32 v14, s22 -; SDAG-NEXT: v_mov_b32_e32 v15, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s16 +; SDAG-NEXT: v_mov_b32_e32 v11, s17 +; SDAG-NEXT: v_mov_b32_e32 v12, s18 +; SDAG-NEXT: v_mov_b32_e32 v13, s19 +; SDAG-NEXT: v_mov_b32_e32 v14, s20 +; SDAG-NEXT: v_mov_b32_e32 v15, s21 +; SDAG-NEXT: v_mov_b32_e32 v16, s22 +; SDAG-NEXT: v_mov_b32_e32 v17, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], 1.0, 0.15915494 op_sel:[1,1,0] op_sel_hi:[1,0,0] +; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[2:9], v[10:17], a[0:3], 1.0, 0.15915494 op_sel:[1,1,0] op_sel_hi:[1,0,0] ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 3 -; SDAG-NEXT: global_store_dwordx4 v16, a[0:3], s[4:5] +; SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[4:5] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll index 91197f9..bc50058 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll @@ -3515,26 +3515,26 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgprs(<8 x i32> inr ; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgprs: ; SDAG: ; %bb.0: ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v24, s0 -; SDAG-NEXT: v_mov_b32_e32 v25, s1 -; SDAG-NEXT: v_mov_b32_e32 v26, s2 -; SDAG-NEXT: v_mov_b32_e32 v27, s3 -; SDAG-NEXT: v_mov_b32_e32 v28, s16 -; SDAG-NEXT: v_mov_b32_e32 v29, s17 -; SDAG-NEXT: v_mov_b32_e32 v30, s18 -; SDAG-NEXT: v_mov_b32_e32 v31, s19 -; SDAG-NEXT: v_mov_b32_e32 v32, s28 -; SDAG-NEXT: v_mov_b32_e32 v33, s29 -; SDAG-NEXT: v_mov_b32_e32 v16, s20 -; SDAG-NEXT: v_mov_b32_e32 v17, s21 -; SDAG-NEXT: v_mov_b32_e32 v18, s22 -; SDAG-NEXT: v_mov_b32_e32 v19, s23 -; SDAG-NEXT: v_mov_b32_e32 v20, s24 -; SDAG-NEXT: v_mov_b32_e32 v21, s25 -; SDAG-NEXT: v_mov_b32_e32 v22, s26 -; SDAG-NEXT: v_mov_b32_e32 v23, s27 -; SDAG-NEXT: v_accvgpr_write_b32 a0, v32 -; SDAG-NEXT: v_accvgpr_write_b32 a1, v33 +; SDAG-NEXT: v_mov_b32_e32 v26, s0 +; SDAG-NEXT: v_mov_b32_e32 v27, s1 +; SDAG-NEXT: v_mov_b32_e32 v28, s2 +; SDAG-NEXT: v_mov_b32_e32 v29, s3 +; SDAG-NEXT: v_mov_b32_e32 v30, s16 +; SDAG-NEXT: v_mov_b32_e32 v31, s17 +; SDAG-NEXT: v_mov_b32_e32 v32, s18 +; SDAG-NEXT: v_mov_b32_e32 v33, s19 +; SDAG-NEXT: v_mov_b32_e32 v16, s28 +; SDAG-NEXT: v_mov_b32_e32 v17, s29 +; SDAG-NEXT: v_mov_b32_e32 v18, s20 +; SDAG-NEXT: v_mov_b32_e32 v19, s21 +; SDAG-NEXT: v_mov_b32_e32 v20, s22 +; SDAG-NEXT: v_mov_b32_e32 v21, s23 +; SDAG-NEXT: v_mov_b32_e32 v22, s24 +; SDAG-NEXT: v_mov_b32_e32 v23, s25 +; SDAG-NEXT: v_mov_b32_e32 v24, s26 +; SDAG-NEXT: v_mov_b32_e32 v25, s27 +; SDAG-NEXT: v_accvgpr_write_b32 a0, v16 +; SDAG-NEXT: v_accvgpr_write_b32 a1, v17 ; SDAG-NEXT: v_accvgpr_write_b32 a2, v0 ; SDAG-NEXT: v_accvgpr_write_b32 a3, v1 ; SDAG-NEXT: v_accvgpr_write_b32 a4, v2 @@ -3550,7 +3550,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgprs(<8 x i32> inr ; SDAG-NEXT: v_accvgpr_write_b32 a14, v12 ; SDAG-NEXT: v_accvgpr_write_b32 a15, v13 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[24:31], v[16:23], a[0:15], v14, v15 op_sel_hi:[0,0,0] +; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[26:33], v[18:25], a[0:15], v14, v15 op_sel_hi:[0,0,0] ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 3 @@ -3993,34 +3993,34 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgpr_vgpr_sgpr__vgp ; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgpr_vgpr_sgpr__vgpr_sgpr: ; SDAG: ; %bb.0: ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v16, s0 -; SDAG-NEXT: v_mov_b32_e32 v17, s1 -; SDAG-NEXT: v_mov_b32_e32 v18, s2 -; SDAG-NEXT: v_mov_b32_e32 v19, s3 -; SDAG-NEXT: v_mov_b32_e32 v20, s16 -; SDAG-NEXT: v_mov_b32_e32 v21, s17 -; SDAG-NEXT: v_mov_b32_e32 v22, s18 -; SDAG-NEXT: v_mov_b32_e32 v23, s19 -; SDAG-NEXT: v_mov_b32_e32 v24, s20 -; SDAG-NEXT: v_mov_b32_e32 v25, s21 -; SDAG-NEXT: v_mov_b32_e32 v26, s22 -; SDAG-NEXT: v_mov_b32_e32 v27, s23 -; SDAG-NEXT: v_mov_b32_e32 v28, s24 -; SDAG-NEXT: v_mov_b32_e32 v29, s25 -; SDAG-NEXT: v_mov_b32_e32 v30, s26 -; SDAG-NEXT: v_mov_b32_e32 v31, s27 -; SDAG-NEXT: v_mov_b32_e32 v32, s28 -; SDAG-NEXT: v_mov_b32_e32 v33, s29 -; SDAG-NEXT: v_accvgpr_write_b32 a0, v24 -; SDAG-NEXT: v_accvgpr_write_b32 a1, v25 -; SDAG-NEXT: v_accvgpr_write_b32 a2, v26 -; SDAG-NEXT: v_accvgpr_write_b32 a3, v27 -; SDAG-NEXT: v_accvgpr_write_b32 a4, v28 -; SDAG-NEXT: v_accvgpr_write_b32 a5, v29 -; SDAG-NEXT: v_accvgpr_write_b32 a6, v30 -; SDAG-NEXT: v_accvgpr_write_b32 a7, v31 -; SDAG-NEXT: v_accvgpr_write_b32 a8, v32 -; SDAG-NEXT: v_accvgpr_write_b32 a9, v33 +; SDAG-NEXT: v_mov_b32_e32 v26, s0 +; SDAG-NEXT: v_mov_b32_e32 v27, s1 +; SDAG-NEXT: v_mov_b32_e32 v28, s2 +; SDAG-NEXT: v_mov_b32_e32 v29, s3 +; SDAG-NEXT: v_mov_b32_e32 v30, s16 +; SDAG-NEXT: v_mov_b32_e32 v31, s17 +; SDAG-NEXT: v_mov_b32_e32 v32, s18 +; SDAG-NEXT: v_mov_b32_e32 v33, s19 +; SDAG-NEXT: v_mov_b32_e32 v16, s20 +; SDAG-NEXT: v_mov_b32_e32 v17, s21 +; SDAG-NEXT: v_mov_b32_e32 v18, s22 +; SDAG-NEXT: v_mov_b32_e32 v19, s23 +; SDAG-NEXT: v_mov_b32_e32 v20, s24 +; SDAG-NEXT: v_mov_b32_e32 v21, s25 +; SDAG-NEXT: v_mov_b32_e32 v22, s26 +; SDAG-NEXT: v_mov_b32_e32 v23, s27 +; SDAG-NEXT: v_mov_b32_e32 v24, s28 +; SDAG-NEXT: v_mov_b32_e32 v25, s29 +; SDAG-NEXT: v_accvgpr_write_b32 a0, v16 +; SDAG-NEXT: v_accvgpr_write_b32 a1, v17 +; SDAG-NEXT: v_accvgpr_write_b32 a2, v18 +; SDAG-NEXT: v_accvgpr_write_b32 a3, v19 +; SDAG-NEXT: v_accvgpr_write_b32 a4, v20 +; SDAG-NEXT: v_accvgpr_write_b32 a5, v21 +; SDAG-NEXT: v_accvgpr_write_b32 a6, v22 +; SDAG-NEXT: v_accvgpr_write_b32 a7, v23 +; SDAG-NEXT: v_accvgpr_write_b32 a8, v24 +; SDAG-NEXT: v_accvgpr_write_b32 a9, v25 ; SDAG-NEXT: v_accvgpr_write_b32 a10, v8 ; SDAG-NEXT: v_accvgpr_write_b32 a11, v9 ; SDAG-NEXT: v_accvgpr_write_b32 a12, v10 @@ -4028,7 +4028,7 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0_sgpr_vgpr_sgpr__vgp ; SDAG-NEXT: v_accvgpr_write_b32 a14, v12 ; SDAG-NEXT: v_accvgpr_write_b32 a15, v13 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[16:23], v[0:7], a[0:15], v14, v15 op_sel_hi:[0,0,0] +; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[26:33], v[0:7], a[0:15], v14, v15 op_sel_hi:[0,0,0] ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 3 @@ -4540,22 +4540,22 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32> ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x80 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_accvgpr_write_b32 a0, s36 -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: v_mov_b32_e32 v4, s12 -; SDAG-NEXT: v_mov_b32_e32 v5, s13 -; SDAG-NEXT: v_mov_b32_e32 v6, s14 -; SDAG-NEXT: v_mov_b32_e32 v7, s15 -; SDAG-NEXT: v_mov_b32_e32 v8, s16 -; SDAG-NEXT: v_mov_b32_e32 v9, s17 -; SDAG-NEXT: v_mov_b32_e32 v10, s18 -; SDAG-NEXT: v_mov_b32_e32 v11, s19 -; SDAG-NEXT: v_mov_b32_e32 v12, s20 -; SDAG-NEXT: v_mov_b32_e32 v13, s21 -; SDAG-NEXT: v_mov_b32_e32 v14, s22 -; SDAG-NEXT: v_mov_b32_e32 v15, s23 +; SDAG-NEXT: v_mov_b32_e32 v2, s8 +; SDAG-NEXT: v_mov_b32_e32 v3, s9 +; SDAG-NEXT: v_mov_b32_e32 v4, s10 +; SDAG-NEXT: v_mov_b32_e32 v5, s11 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 +; SDAG-NEXT: v_mov_b32_e32 v10, s16 +; SDAG-NEXT: v_mov_b32_e32 v11, s17 +; SDAG-NEXT: v_mov_b32_e32 v12, s18 +; SDAG-NEXT: v_mov_b32_e32 v13, s19 +; SDAG-NEXT: v_mov_b32_e32 v14, s20 +; SDAG-NEXT: v_mov_b32_e32 v15, s21 +; SDAG-NEXT: v_mov_b32_e32 v16, s22 +; SDAG-NEXT: v_mov_b32_e32 v17, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s37 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s38 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s39 @@ -4571,9 +4571,9 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32> ; SDAG-NEXT: v_accvgpr_write_b32 a13, s49 ; SDAG-NEXT: v_accvgpr_write_b32 a14, s50 ; SDAG-NEXT: v_accvgpr_write_b32 a15, s51 -; SDAG-NEXT: v_mov_b32_e32 v16, s1 +; SDAG-NEXT: v_mov_b32_e32 v0, s1 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], s0, v16 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2 +; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[2:9], v[10:17], a[0:15], s0, v0 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2 ; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 7 @@ -4735,26 +4735,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x ; SDAG: ; %bb.0: ; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: v_mov_b32_e32 v4, s16 -; SDAG-NEXT: v_mov_b32_e32 v5, s17 -; SDAG-NEXT: v_mov_b32_e32 v6, s18 -; SDAG-NEXT: v_mov_b32_e32 v7, s19 -; SDAG-NEXT: v_mov_b32_e32 v8, s20 -; SDAG-NEXT: v_mov_b32_e32 v9, s21 -; SDAG-NEXT: v_mov_b32_e32 v10, s22 -; SDAG-NEXT: v_mov_b32_e32 v11, s23 +; SDAG-NEXT: v_mov_b32_e32 v2, s12 +; SDAG-NEXT: v_mov_b32_e32 v3, s13 +; SDAG-NEXT: v_mov_b32_e32 v4, s14 +; SDAG-NEXT: v_mov_b32_e32 v5, s15 +; SDAG-NEXT: v_mov_b32_e32 v6, s16 +; SDAG-NEXT: v_mov_b32_e32 v7, s17 +; SDAG-NEXT: v_mov_b32_e32 v8, s18 +; SDAG-NEXT: v_mov_b32_e32 v9, s19 +; SDAG-NEXT: v_mov_b32_e32 v10, s20 +; SDAG-NEXT: v_mov_b32_e32 v11, s21 +; SDAG-NEXT: v_mov_b32_e32 v12, s22 +; SDAG-NEXT: v_mov_b32_e32 v13, s23 ; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40 ; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80 -; SDAG-NEXT: v_mov_b32_e32 v12, s24 -; SDAG-NEXT: v_mov_b32_e32 v13, s25 -; SDAG-NEXT: v_mov_b32_e32 v14, s26 +; SDAG-NEXT: v_mov_b32_e32 v14, s24 +; SDAG-NEXT: v_mov_b32_e32 v15, s25 +; SDAG-NEXT: v_mov_b32_e32 v16, s26 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_accvgpr_write_b32 a0, s8 -; SDAG-NEXT: v_mov_b32_e32 v15, s27 +; SDAG-NEXT: v_mov_b32_e32 v17, s27 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s11 @@ -4770,45 +4770,44 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x ; SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; SDAG-NEXT: v_accvgpr_write_b32 a15, s23 -; SDAG-NEXT: v_mov_b32_e32 v16, s1 +; SDAG-NEXT: v_mov_b32_e32 v0, s1 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], s0, v16 op_sel_hi:[0,0,0] -; SDAG-NEXT: v_mov_b32_e32 v0, s20 -; SDAG-NEXT: v_mov_b32_e32 v1, s21 -; SDAG-NEXT: v_mov_b32_e32 v2, s22 -; SDAG-NEXT: v_mov_b32_e32 v3, s23 -; SDAG-NEXT: v_mov_b64_e32 v[4:5], 48 -; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[2:9], v[10:17], a[0:15], s0, v0 op_sel_hi:[0,0,0] +; SDAG-NEXT: v_mov_b32_e32 v2, s20 +; SDAG-NEXT: v_mov_b32_e32 v3, s21 +; SDAG-NEXT: v_mov_b32_e32 v4, s22 +; SDAG-NEXT: v_mov_b32_e32 v5, s23 +; SDAG-NEXT: v_mov_b64_e32 v[0:1], 48 +; SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: v_mov_b64_e32 v[6:7], 32 -; SDAG-NEXT: v_mov_b64_e32 v[8:9], 16 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v6, s18 +; SDAG-NEXT: v_mov_b32_e32 v7, s19 +; SDAG-NEXT: v_mov_b32_e32 v4, s16 +; SDAG-NEXT: v_mov_b32_e32 v5, s17 +; SDAG-NEXT: v_mov_b64_e32 v[2:3], 32 +; SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: v_mov_b64_e32 v[10:11], 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b64_e32 v[4:5], 16 +; SDAG-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v10, s10 +; SDAG-NEXT: v_mov_b32_e32 v11, s11 +; SDAG-NEXT: v_mov_b32_e32 v8, s8 +; SDAG-NEXT: v_mov_b32_e32 v9, s9 +; SDAG-NEXT: v_mov_b64_e32 v[6:7], 0 +; SDAG-NEXT: global_store_dwordx4 v[6:7], v[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[6:7], a[8:11], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[4:5], a[12:15], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[10:11], a[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[8:9], a[4:7], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; @@ -4922,42 +4921,41 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8 ; SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; SDAG-NEXT: s_nop 1 ; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], 25, 42 op_sel_hi:[0,0,0] blgp:2 -; SDAG-NEXT: v_mov_b32_e32 v0, s20 -; SDAG-NEXT: v_mov_b32_e32 v1, s21 -; SDAG-NEXT: v_mov_b32_e32 v2, s22 -; SDAG-NEXT: v_mov_b32_e32 v3, s23 -; SDAG-NEXT: v_mov_b64_e32 v[4:5], 48 -; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v2, s20 +; SDAG-NEXT: v_mov_b32_e32 v3, s21 +; SDAG-NEXT: v_mov_b32_e32 v4, s22 +; SDAG-NEXT: v_mov_b32_e32 v5, s23 +; SDAG-NEXT: v_mov_b64_e32 v[0:1], 48 +; SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: v_mov_b64_e32 v[6:7], 32 -; SDAG-NEXT: v_mov_b64_e32 v[8:9], 16 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v6, s18 +; SDAG-NEXT: v_mov_b32_e32 v7, s19 +; SDAG-NEXT: v_mov_b32_e32 v4, s16 +; SDAG-NEXT: v_mov_b32_e32 v5, s17 +; SDAG-NEXT: v_mov_b64_e32 v[2:3], 32 +; SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: v_mov_b64_e32 v[10:11], 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b64_e32 v[4:5], 16 +; SDAG-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v10, s10 +; SDAG-NEXT: v_mov_b32_e32 v11, s11 +; SDAG-NEXT: v_mov_b32_e32 v8, s8 +; SDAG-NEXT: v_mov_b32_e32 v9, s9 +; SDAG-NEXT: v_mov_b64_e32 v[6:7], 0 +; SDAG-NEXT: global_store_dwordx4 v[6:7], v[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[6:7], a[8:11], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[4:5], a[12:15], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[10:11], a[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[8:9], a[4:7], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; @@ -5069,42 +5067,41 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma ; SDAG-NEXT: v_accvgpr_write_b32 a16, s8 ; SDAG-NEXT: s_nop 1 ; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[16:31] blgp:2 -; SDAG-NEXT: v_mov_b32_e32 v0, s20 -; SDAG-NEXT: v_mov_b32_e32 v1, s21 -; SDAG-NEXT: v_mov_b32_e32 v2, s22 -; SDAG-NEXT: v_mov_b32_e32 v3, s23 -; SDAG-NEXT: v_mov_b64_e32 v[4:5], 48 -; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v2, s20 +; SDAG-NEXT: v_mov_b32_e32 v3, s21 +; SDAG-NEXT: v_mov_b32_e32 v4, s22 +; SDAG-NEXT: v_mov_b32_e32 v5, s23 +; SDAG-NEXT: v_mov_b64_e32 v[0:1], 48 +; SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: v_mov_b64_e32 v[6:7], 32 -; SDAG-NEXT: v_mov_b64_e32 v[8:9], 16 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v6, s18 +; SDAG-NEXT: v_mov_b32_e32 v7, s19 +; SDAG-NEXT: v_mov_b32_e32 v4, s16 +; SDAG-NEXT: v_mov_b32_e32 v5, s17 +; SDAG-NEXT: v_mov_b64_e32 v[2:3], 32 +; SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: v_mov_b64_e32 v[10:11], 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b64_e32 v[4:5], 16 +; SDAG-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v10, s10 +; SDAG-NEXT: v_mov_b32_e32 v11, s11 +; SDAG-NEXT: v_mov_b32_e32 v8, s8 +; SDAG-NEXT: v_mov_b32_e32 v9, s9 +; SDAG-NEXT: v_mov_b64_e32 v[6:7], 0 +; SDAG-NEXT: global_store_dwordx4 v[6:7], v[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[6:7], a[8:11], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[4:5], a[12:15], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[10:11], a[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[8:9], a[4:7], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; @@ -5216,42 +5213,41 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non ; SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; SDAG-NEXT: s_nop 1 ; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], 25, 42 op_sel_hi:[0,0,0] blgp:2 -; SDAG-NEXT: v_mov_b32_e32 v0, s20 -; SDAG-NEXT: v_mov_b32_e32 v1, s21 -; SDAG-NEXT: v_mov_b32_e32 v2, s22 -; SDAG-NEXT: v_mov_b32_e32 v3, s23 -; SDAG-NEXT: v_mov_b64_e32 v[4:5], 48 -; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v2, s20 +; SDAG-NEXT: v_mov_b32_e32 v3, s21 +; SDAG-NEXT: v_mov_b32_e32 v4, s22 +; SDAG-NEXT: v_mov_b32_e32 v5, s23 +; SDAG-NEXT: v_mov_b64_e32 v[0:1], 48 +; SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: v_mov_b64_e32 v[6:7], 32 -; SDAG-NEXT: v_mov_b64_e32 v[8:9], 16 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v6, s18 +; SDAG-NEXT: v_mov_b32_e32 v7, s19 +; SDAG-NEXT: v_mov_b32_e32 v4, s16 +; SDAG-NEXT: v_mov_b32_e32 v5, s17 +; SDAG-NEXT: v_mov_b64_e32 v[2:3], 32 +; SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: v_mov_b64_e32 v[10:11], 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 -; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v8, s14 +; SDAG-NEXT: v_mov_b32_e32 v9, s15 +; SDAG-NEXT: v_mov_b32_e32 v6, s12 +; SDAG-NEXT: v_mov_b32_e32 v7, s13 +; SDAG-NEXT: v_mov_b64_e32 v[4:5], 16 +; SDAG-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_mov_b32_e32 v0, s8 -; SDAG-NEXT: v_mov_b32_e32 v1, s9 -; SDAG-NEXT: v_mov_b32_e32 v2, s10 -; SDAG-NEXT: v_mov_b32_e32 v3, s11 -; SDAG-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; SDAG-NEXT: v_mov_b32_e32 v10, s10 +; SDAG-NEXT: v_mov_b32_e32 v11, s11 +; SDAG-NEXT: v_mov_b32_e32 v8, s8 +; SDAG-NEXT: v_mov_b32_e32 v9, s9 +; SDAG-NEXT: v_mov_b64_e32 v[6:7], 0 +; SDAG-NEXT: global_store_dwordx4 v[6:7], v[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[6:7], a[8:11], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[4:5], a[12:15], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[10:11], a[0:3], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: global_store_dwordx4 v[8:9], a[4:7], off sc0 sc1 +; SDAG-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_endpgm ; diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll index 7193fee..ea9334a 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll @@ -1,22 +1,100 @@ -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -global-isel < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -stress-regalloc=10 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -stress-regalloc=10 -global-isel < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-SDAG %s +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-GISEL %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 -stress-regalloc=10 < %s | FileCheck --check-prefixes=GFX942-STRESS,GFX942-SDAG-STRESS %s +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 -stress-regalloc=10 < %s | FileCheck --check-prefixes=GFX942-STRESS,GFX942-GISEL-STRESS %s declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x8.xf32(<2 x float>, <2 x float>, <4 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x4.xf32(<2 x float>, <2 x float>, <16 x float>, i32, i32, i32) -; GCN-LABEL: {{^}}test_mfma_f32_16x16x8xf32: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1.0 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2.0 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 0x40400000 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4.0 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_16x16x8_xf32 a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:[[TWO]]], v[[[THREE]]:[[FOUR]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_16x16x8_xf32 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_16x16x8xf32(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_f32_16x16x8xf32: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 1.0 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 2.0 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0x40400000 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 4.0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_f32_16x16x8_xf32 a[0:3], v[4:5], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: s_nop 6 +; GFX942-SDAG-NEXT: global_store_dwordx4 v2, a[0:3], s[6:7] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_16x16x8xf32: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-NEXT: s_mov_b32 s4, 1.0 +; GFX942-GISEL-NEXT: s_mov_b32 s5, 2.0 +; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX942-GISEL-NEXT: s_mov_b32 s4, 0x40400000 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-NEXT: s_mov_b32 s5, 4.0 +; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5] +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_16x16x8_xf32 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 5 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-SDAG-STRESS-LABEL: test_mfma_f32_16x16x8xf32: +; GFX942-SDAG-STRESS: ; %bb.0: ; %bb +; GFX942-SDAG-STRESS-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-SDAG-STRESS-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX942-SDAG-STRESS-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX942-SDAG-STRESS-NEXT: v_mov_b32_e32 v2, 0x40400000 +; GFX942-SDAG-STRESS-NEXT: v_mov_b32_e32 v3, 4.0 +; GFX942-SDAG-STRESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-STRESS-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-SDAG-STRESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-SDAG-STRESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-STRESS-NEXT: s_nop 1 +; GFX942-SDAG-STRESS-NEXT: v_mfma_f32_16x16x8_xf32 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-STRESS-NEXT: s_nop 6 +; GFX942-SDAG-STRESS-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX942-SDAG-STRESS-NEXT: s_endpgm +; +; GFX942-GISEL-STRESS-LABEL: test_mfma_f32_16x16x8xf32: +; GFX942-GISEL-STRESS: ; %bb.0: ; %bb +; GFX942-GISEL-STRESS-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-STRESS-NEXT: s_mov_b32 s0, 1.0 +; GFX942-GISEL-STRESS-NEXT: s_mov_b32 s2, 0x40400000 +; GFX942-GISEL-STRESS-NEXT: s_mov_b32 s1, 2.0 +; GFX942-GISEL-STRESS-NEXT: s_mov_b32 s3, 4.0 +; GFX942-GISEL-STRESS-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-GISEL-STRESS-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-GISEL-STRESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-STRESS-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-STRESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-STRESS-NEXT: s_nop 1 +; GFX942-GISEL-STRESS-NEXT: v_mfma_f32_16x16x8_xf32 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-STRESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-STRESS-NEXT: s_nop 5 +; GFX942-GISEL-STRESS-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-STRESS-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x8.xf32(<2 x float> <float 1.0, float 2.0>, <2 x float> <float 3.0, float 4.0>, <4 x float> %in.1, i32 1, i32 2, i32 3) @@ -24,17 +102,158 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x4xf32: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1.0 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2.0 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 0x40400000 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4.0 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_32x32x4_xf32 a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:[[TWO]]], v[[[THREE]]:[[FOUR]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_32x32x4_xf32 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x4xf32(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_f32_32x32x4xf32: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 1.0 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 2.0 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0x40400000 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 4.0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_f32_32x32x4_xf32 a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-NEXT: s_nop 7 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_32x32x4xf32: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-NEXT: s_mov_b32 s18, 1.0 +; GFX942-GISEL-NEXT: s_mov_b32 s19, 2.0 +; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[18:19] +; GFX942-GISEL-NEXT: s_mov_b32 s18, 0x40400000 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-NEXT: s_mov_b32 s19, 4.0 +; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19] +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_32x32x4_xf32 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 7 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-SDAG-STRESS-LABEL: test_mfma_f32_32x32x4xf32: +; GFX942-SDAG-STRESS: ; %bb.0: ; %bb +; GFX942-SDAG-STRESS-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-STRESS-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX942-SDAG-STRESS-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX942-SDAG-STRESS-NEXT: v_mov_b32_e32 v2, 0x40400000 +; GFX942-SDAG-STRESS-NEXT: v_mov_b32_e32 v3, 4.0 +; GFX942-SDAG-STRESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-STRESS-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-STRESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-STRESS-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-STRESS-NEXT: s_nop 1 +; GFX942-SDAG-STRESS-NEXT: v_mfma_f32_32x32x4_xf32 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-STRESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-STRESS-NEXT: s_nop 7 +; GFX942-SDAG-STRESS-NEXT: s_nop 1 +; GFX942-SDAG-STRESS-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-STRESS-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-STRESS-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-STRESS-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-STRESS-NEXT: s_endpgm +; +; GFX942-GISEL-STRESS-LABEL: test_mfma_f32_32x32x4xf32: +; GFX942-GISEL-STRESS: ; %bb.0: ; %bb +; GFX942-GISEL-STRESS-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-STRESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-STRESS-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-STRESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-STRESS-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-STRESS-NEXT: s_mov_b32 s0, 1.0 +; GFX942-GISEL-STRESS-NEXT: s_mov_b32 s1, 2.0 +; GFX942-GISEL-STRESS-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-GISEL-STRESS-NEXT: s_mov_b32 s0, 0x40400000 +; GFX942-GISEL-STRESS-NEXT: s_mov_b32 s1, 4.0 +; GFX942-GISEL-STRESS-NEXT: v_mov_b64_e32 v[2:3], s[0:1] +; GFX942-GISEL-STRESS-NEXT: s_nop 1 +; GFX942-GISEL-STRESS-NEXT: v_mfma_f32_32x32x4_xf32 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-STRESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-STRESS-NEXT: s_nop 7 +; GFX942-GISEL-STRESS-NEXT: s_nop 1 +; GFX942-GISEL-STRESS-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-STRESS-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-STRESS-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-STRESS-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-STRESS-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x4.xf32(<2 x float> <float 1.0, float 2.0>, <2 x float> <float 3.0, float 4.0>, <16 x float> %in.1, i32 1, i32 2, i32 3) @@ -43,3 +262,6 @@ bb: } attributes #0 = { "amdgpu-flat-work-group-size"="1,256" } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GFX942: {{.*}} +; GFX942-STRESS: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll index 77d4aad..8056881 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll @@ -17,24 +17,24 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x64_f16__vgpr(ptr addrspace(1) % ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34 ; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v0 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 +; SDAG-NEXT: v_mov_b32_e32 v12, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: global_load_dwordx4 v[8:11], v0, s[6:7] +; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7] ; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44 ; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64 -; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[2:3] -; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[0:1] +; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13] ; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15] -; SDAG-NEXT: v_mov_b32_e32 v17, s16 +; SDAG-NEXT: v_mov_b32_e32 v13, s16 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_smfmac_f32_16x16x64_f16 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2 +; SDAG-NEXT: v_smfmac_f32_16x16x64_f16 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2 ; SDAG-NEXT: s_nop 7 -; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] +; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_smfmac_f32_16x16x64_f16__vgpr: @@ -44,23 +44,23 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x64_f16__vgpr(ptr addrspace(1) % ; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[6:7] +; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7] ; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44 ; GISEL-NEXT: s_load_dword s16, s[4:5], 0x64 -; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[2:3] -; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1] +; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] ; GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13] ; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15] -; GISEL-NEXT: v_mov_b32_e32 v16, s16 +; GISEL-NEXT: v_mov_b32_e32 v12, s16 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_nop 0 -; GISEL-NEXT: v_smfmac_f32_16x16x64_f16 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2 +; GISEL-NEXT: v_smfmac_f32_16x16x64_f16 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2 ; GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GISEL-NEXT: s_nop 6 -; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[6:7] +; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[6:7] ; GISEL-NEXT: s_endpgm bb: %id = call i32 @llvm.amdgcn.workitem.id.x() @@ -120,25 +120,25 @@ define <4 x float> @test_smfmac_f32_16x16x64_f16__sgpr(<8 x half> inreg %arg0, < ; SDAG-LABEL: test_smfmac_f32_16x16x64_f16__sgpr: ; SDAG: ; %bb.0: ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v8, s0 -; SDAG-NEXT: v_mov_b32_e32 v9, s1 -; SDAG-NEXT: v_mov_b32_e32 v10, s2 -; SDAG-NEXT: v_mov_b32_e32 v11, s3 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: v_mov_b32_e32 v4, s20 -; SDAG-NEXT: v_mov_b32_e32 v5, s21 -; SDAG-NEXT: v_mov_b32_e32 v6, s22 -; SDAG-NEXT: v_mov_b32_e32 v7, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s0 +; SDAG-NEXT: v_mov_b32_e32 v11, s1 +; SDAG-NEXT: v_mov_b32_e32 v12, s2 +; SDAG-NEXT: v_mov_b32_e32 v13, s3 +; SDAG-NEXT: v_mov_b32_e32 v2, s16 +; SDAG-NEXT: v_mov_b32_e32 v3, s17 +; SDAG-NEXT: v_mov_b32_e32 v4, s18 +; SDAG-NEXT: v_mov_b32_e32 v5, s19 +; SDAG-NEXT: v_mov_b32_e32 v6, s20 +; SDAG-NEXT: v_mov_b32_e32 v7, s21 +; SDAG-NEXT: v_mov_b32_e32 v8, s22 +; SDAG-NEXT: v_mov_b32_e32 v9, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s24 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s25 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s26 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s27 -; SDAG-NEXT: v_mov_b32_e32 v12, s28 +; SDAG-NEXT: v_mov_b32_e32 v0, s28 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_smfmac_f32_16x16x64_f16 a[0:3], v[8:11], v[0:7], v12 +; SDAG-NEXT: v_smfmac_f32_16x16x64_f16 a[0:3], v[10:13], v[2:9], v0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: v_accvgpr_read_b32 v0, a0 ; SDAG-NEXT: v_accvgpr_read_b32 v1, a1 @@ -547,24 +547,24 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x64_bf16__vgpr(ptr addrspace(1) ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34 ; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GCN-NEXT: v_lshlrev_b32_e32 v0, 4, v0 -; GCN-NEXT: v_mov_b32_e32 v16, 0 +; GCN-NEXT: v_mov_b32_e32 v12, 0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: global_load_dwordx4 v[8:11], v0, s[6:7] +; GCN-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7] ; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44 ; GCN-NEXT: s_load_dword s16, s[4:5], 0x64 -; GCN-NEXT: v_mov_b64_e32 v[14:15], s[2:3] -; GCN-NEXT: v_mov_b64_e32 v[12:13], s[0:1] +; GCN-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GCN-NEXT: v_mov_b64_e32 v[8:9], s[0:1] ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GCN-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GCN-NEXT: v_mov_b64_e32 v[4:5], s[12:13] ; GCN-NEXT: v_mov_b64_e32 v[6:7], s[14:15] -; GCN-NEXT: v_mov_b32_e32 v17, s16 +; GCN-NEXT: v_mov_b32_e32 v13, s16 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_nop 0 -; GCN-NEXT: v_smfmac_f32_16x16x64_bf16 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2 +; GCN-NEXT: v_smfmac_f32_16x16x64_bf16 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2 ; GCN-NEXT: s_nop 7 -; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] +; GCN-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7] ; GCN-NEXT: s_endpgm bb: %id = call i32 @llvm.amdgcn.workitem.id.x() @@ -624,25 +624,25 @@ define <4 x float> @test_smfmac_f32_16x16x64_bf16__sgpr(<8 x bfloat> inreg %arg0 ; GCN-LABEL: test_smfmac_f32_16x16x64_bf16__sgpr: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: v_mov_b32_e32 v8, s0 -; GCN-NEXT: v_mov_b32_e32 v9, s1 -; GCN-NEXT: v_mov_b32_e32 v10, s2 -; GCN-NEXT: v_mov_b32_e32 v11, s3 -; GCN-NEXT: v_mov_b32_e32 v0, s16 -; GCN-NEXT: v_mov_b32_e32 v1, s17 -; GCN-NEXT: v_mov_b32_e32 v2, s18 -; GCN-NEXT: v_mov_b32_e32 v3, s19 -; GCN-NEXT: v_mov_b32_e32 v4, s20 -; GCN-NEXT: v_mov_b32_e32 v5, s21 -; GCN-NEXT: v_mov_b32_e32 v6, s22 -; GCN-NEXT: v_mov_b32_e32 v7, s23 +; GCN-NEXT: v_mov_b32_e32 v10, s0 +; GCN-NEXT: v_mov_b32_e32 v11, s1 +; GCN-NEXT: v_mov_b32_e32 v12, s2 +; GCN-NEXT: v_mov_b32_e32 v13, s3 +; GCN-NEXT: v_mov_b32_e32 v2, s16 +; GCN-NEXT: v_mov_b32_e32 v3, s17 +; GCN-NEXT: v_mov_b32_e32 v4, s18 +; GCN-NEXT: v_mov_b32_e32 v5, s19 +; GCN-NEXT: v_mov_b32_e32 v6, s20 +; GCN-NEXT: v_mov_b32_e32 v7, s21 +; GCN-NEXT: v_mov_b32_e32 v8, s22 +; GCN-NEXT: v_mov_b32_e32 v9, s23 ; GCN-NEXT: v_accvgpr_write_b32 a0, s24 ; GCN-NEXT: v_accvgpr_write_b32 a1, s25 ; GCN-NEXT: v_accvgpr_write_b32 a2, s26 ; GCN-NEXT: v_accvgpr_write_b32 a3, s27 -; GCN-NEXT: v_mov_b32_e32 v12, s28 +; GCN-NEXT: v_mov_b32_e32 v0, s28 ; GCN-NEXT: s_nop 1 -; GCN-NEXT: v_smfmac_f32_16x16x64_bf16 a[0:3], v[8:11], v[0:7], v12 +; GCN-NEXT: v_smfmac_f32_16x16x64_bf16 a[0:3], v[10:13], v[2:9], v0 ; GCN-NEXT: s_nop 7 ; GCN-NEXT: v_accvgpr_read_b32 v0, a0 ; GCN-NEXT: v_accvgpr_read_b32 v1, a1 @@ -855,30 +855,30 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) % ; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: global_load_dwordx4 v[8:11], v0, s[6:7] +; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7] ; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 -; SDAG-NEXT: v_mov_b32_e32 v12, s8 -; SDAG-NEXT: v_mov_b32_e32 v13, s9 -; SDAG-NEXT: v_mov_b32_e32 v14, s10 -; SDAG-NEXT: v_mov_b32_e32 v15, s11 -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 +; SDAG-NEXT: v_mov_b32_e32 v14, s8 +; SDAG-NEXT: v_mov_b32_e32 v15, s9 +; SDAG-NEXT: v_mov_b32_e32 v16, s10 +; SDAG-NEXT: v_mov_b32_e32 v17, s11 +; SDAG-NEXT: v_mov_b32_e32 v2, s12 +; SDAG-NEXT: v_mov_b32_e32 v3, s13 +; SDAG-NEXT: v_mov_b32_e32 v4, s14 +; SDAG-NEXT: v_mov_b32_e32 v5, s15 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v4, s0 -; SDAG-NEXT: v_mov_b32_e32 v5, s1 -; SDAG-NEXT: v_mov_b32_e32 v6, s2 -; SDAG-NEXT: v_mov_b32_e32 v7, s3 -; SDAG-NEXT: v_mov_b32_e32 v17, s16 +; SDAG-NEXT: v_mov_b32_e32 v6, s0 +; SDAG-NEXT: v_mov_b32_e32 v7, s1 +; SDAG-NEXT: v_mov_b32_e32 v8, s2 +; SDAG-NEXT: v_mov_b32_e32 v9, s3 +; SDAG-NEXT: v_mov_b32_e32 v1, s16 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_smfmac_i32_16x16x128_i8 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2 +; SDAG-NEXT: v_smfmac_i32_16x16x128_i8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2 ; SDAG-NEXT: s_nop 7 -; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] +; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_smfmac_i32_16x16x128_i8__vgpr: @@ -887,24 +887,24 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) % ; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1] +; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1] ; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54 ; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11] -; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17] ; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19] -; GISEL-NEXT: v_mov_b32_e32 v16, s2 +; GISEL-NEXT: v_mov_b32_e32 v12, s2 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_nop 0 -; GISEL-NEXT: v_smfmac_i32_16x16x128_i8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2 +; GISEL-NEXT: v_smfmac_i32_16x16x128_i8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2 ; GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GISEL-NEXT: s_nop 6 -; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1] +; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1] ; GISEL-NEXT: s_endpgm bb: %id = call i32 @llvm.amdgcn.workitem.id.x() @@ -964,25 +964,25 @@ define <4 x i32> @test_smfmac_i32_16x16x128_i8__sgpr(<4 x i32> inreg %arg0, <8 x ; SDAG-LABEL: test_smfmac_i32_16x16x128_i8__sgpr: ; SDAG: ; %bb.0: ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v8, s0 -; SDAG-NEXT: v_mov_b32_e32 v9, s1 -; SDAG-NEXT: v_mov_b32_e32 v10, s2 -; SDAG-NEXT: v_mov_b32_e32 v11, s3 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: v_mov_b32_e32 v4, s20 -; SDAG-NEXT: v_mov_b32_e32 v5, s21 -; SDAG-NEXT: v_mov_b32_e32 v6, s22 -; SDAG-NEXT: v_mov_b32_e32 v7, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s0 +; SDAG-NEXT: v_mov_b32_e32 v11, s1 +; SDAG-NEXT: v_mov_b32_e32 v12, s2 +; SDAG-NEXT: v_mov_b32_e32 v13, s3 +; SDAG-NEXT: v_mov_b32_e32 v2, s16 +; SDAG-NEXT: v_mov_b32_e32 v3, s17 +; SDAG-NEXT: v_mov_b32_e32 v4, s18 +; SDAG-NEXT: v_mov_b32_e32 v5, s19 +; SDAG-NEXT: v_mov_b32_e32 v6, s20 +; SDAG-NEXT: v_mov_b32_e32 v7, s21 +; SDAG-NEXT: v_mov_b32_e32 v8, s22 +; SDAG-NEXT: v_mov_b32_e32 v9, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s24 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s25 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s26 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s27 -; SDAG-NEXT: v_mov_b32_e32 v12, s28 +; SDAG-NEXT: v_mov_b32_e32 v0, s28 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_smfmac_i32_16x16x128_i8 a[0:3], v[8:11], v[0:7], v12 +; SDAG-NEXT: v_smfmac_i32_16x16x128_i8 a[0:3], v[10:13], v[2:9], v0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: v_accvgpr_read_b32 v0, a0 ; SDAG-NEXT: v_accvgpr_read_b32 v1, a1 @@ -1032,22 +1032,22 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a ; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v24, s8 -; SDAG-NEXT: v_mov_b32_e32 v25, s9 -; SDAG-NEXT: v_mov_b32_e32 v26, s10 -; SDAG-NEXT: v_mov_b32_e32 v27, s11 -; SDAG-NEXT: v_mov_b32_e32 v16, s12 -; SDAG-NEXT: v_mov_b32_e32 v17, s13 -; SDAG-NEXT: v_mov_b32_e32 v18, s14 -; SDAG-NEXT: v_mov_b32_e32 v19, s15 -; SDAG-NEXT: v_mov_b32_e32 v20, s0 -; SDAG-NEXT: v_mov_b32_e32 v21, s1 -; SDAG-NEXT: v_mov_b32_e32 v22, s2 -; SDAG-NEXT: v_mov_b32_e32 v23, s3 -; SDAG-NEXT: v_mov_b32_e32 v28, s16 +; SDAG-NEXT: v_mov_b32_e32 v26, s8 +; SDAG-NEXT: v_mov_b32_e32 v27, s9 +; SDAG-NEXT: v_mov_b32_e32 v28, s10 +; SDAG-NEXT: v_mov_b32_e32 v29, s11 +; SDAG-NEXT: v_mov_b32_e32 v18, s12 +; SDAG-NEXT: v_mov_b32_e32 v19, s13 +; SDAG-NEXT: v_mov_b32_e32 v20, s14 +; SDAG-NEXT: v_mov_b32_e32 v21, s15 +; SDAG-NEXT: v_mov_b32_e32 v22, s0 +; SDAG-NEXT: v_mov_b32_e32 v23, s1 +; SDAG-NEXT: v_mov_b32_e32 v24, s2 +; SDAG-NEXT: v_mov_b32_e32 v25, s3 +; SDAG-NEXT: v_mov_b32_e32 v16, s16 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2 +; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2 ; SDAG-NEXT: v_mov_b32_e32 v16, 0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 2 @@ -1397,30 +1397,30 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace ; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: global_load_dwordx4 v[8:11], v0, s[6:7] +; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7] ; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 -; SDAG-NEXT: v_mov_b32_e32 v12, s8 -; SDAG-NEXT: v_mov_b32_e32 v13, s9 -; SDAG-NEXT: v_mov_b32_e32 v14, s10 -; SDAG-NEXT: v_mov_b32_e32 v15, s11 -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 +; SDAG-NEXT: v_mov_b32_e32 v14, s8 +; SDAG-NEXT: v_mov_b32_e32 v15, s9 +; SDAG-NEXT: v_mov_b32_e32 v16, s10 +; SDAG-NEXT: v_mov_b32_e32 v17, s11 +; SDAG-NEXT: v_mov_b32_e32 v2, s12 +; SDAG-NEXT: v_mov_b32_e32 v3, s13 +; SDAG-NEXT: v_mov_b32_e32 v4, s14 +; SDAG-NEXT: v_mov_b32_e32 v5, s15 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v4, s0 -; SDAG-NEXT: v_mov_b32_e32 v5, s1 -; SDAG-NEXT: v_mov_b32_e32 v6, s2 -; SDAG-NEXT: v_mov_b32_e32 v7, s3 -; SDAG-NEXT: v_mov_b32_e32 v17, s16 +; SDAG-NEXT: v_mov_b32_e32 v6, s0 +; SDAG-NEXT: v_mov_b32_e32 v7, s1 +; SDAG-NEXT: v_mov_b32_e32 v8, s2 +; SDAG-NEXT: v_mov_b32_e32 v9, s3 +; SDAG-NEXT: v_mov_b32_e32 v1, s16 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2 +; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2 ; SDAG-NEXT: s_nop 7 -; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] +; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__vgpr: @@ -1429,24 +1429,24 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace ; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1] +; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1] ; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54 ; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11] -; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17] ; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19] -; GISEL-NEXT: v_mov_b32_e32 v16, s2 +; GISEL-NEXT: v_mov_b32_e32 v12, s2 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_nop 0 -; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2 +; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2 ; GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GISEL-NEXT: s_nop 6 -; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1] +; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1] ; GISEL-NEXT: s_endpgm bb: %id = call i32 @llvm.amdgcn.workitem.id.x() @@ -1506,25 +1506,25 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__sgpr(<4 x i32> inreg %arg ; SDAG-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__sgpr: ; SDAG: ; %bb.0: ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v8, s0 -; SDAG-NEXT: v_mov_b32_e32 v9, s1 -; SDAG-NEXT: v_mov_b32_e32 v10, s2 -; SDAG-NEXT: v_mov_b32_e32 v11, s3 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: v_mov_b32_e32 v4, s20 -; SDAG-NEXT: v_mov_b32_e32 v5, s21 -; SDAG-NEXT: v_mov_b32_e32 v6, s22 -; SDAG-NEXT: v_mov_b32_e32 v7, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s0 +; SDAG-NEXT: v_mov_b32_e32 v11, s1 +; SDAG-NEXT: v_mov_b32_e32 v12, s2 +; SDAG-NEXT: v_mov_b32_e32 v13, s3 +; SDAG-NEXT: v_mov_b32_e32 v2, s16 +; SDAG-NEXT: v_mov_b32_e32 v3, s17 +; SDAG-NEXT: v_mov_b32_e32 v4, s18 +; SDAG-NEXT: v_mov_b32_e32 v5, s19 +; SDAG-NEXT: v_mov_b32_e32 v6, s20 +; SDAG-NEXT: v_mov_b32_e32 v7, s21 +; SDAG-NEXT: v_mov_b32_e32 v8, s22 +; SDAG-NEXT: v_mov_b32_e32 v9, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s24 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s25 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s26 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s27 -; SDAG-NEXT: v_mov_b32_e32 v12, s28 +; SDAG-NEXT: v_mov_b32_e32 v0, s28 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 a[0:3], v[8:11], v[0:7], v12 +; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 a[0:3], v[10:13], v[2:9], v0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: v_accvgpr_read_b32 v0, a0 ; SDAG-NEXT: v_accvgpr_read_b32 v1, a1 @@ -1566,30 +1566,30 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace ; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: global_load_dwordx4 v[8:11], v0, s[6:7] +; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7] ; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 -; SDAG-NEXT: v_mov_b32_e32 v12, s8 -; SDAG-NEXT: v_mov_b32_e32 v13, s9 -; SDAG-NEXT: v_mov_b32_e32 v14, s10 -; SDAG-NEXT: v_mov_b32_e32 v15, s11 -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 +; SDAG-NEXT: v_mov_b32_e32 v14, s8 +; SDAG-NEXT: v_mov_b32_e32 v15, s9 +; SDAG-NEXT: v_mov_b32_e32 v16, s10 +; SDAG-NEXT: v_mov_b32_e32 v17, s11 +; SDAG-NEXT: v_mov_b32_e32 v2, s12 +; SDAG-NEXT: v_mov_b32_e32 v3, s13 +; SDAG-NEXT: v_mov_b32_e32 v4, s14 +; SDAG-NEXT: v_mov_b32_e32 v5, s15 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v4, s0 -; SDAG-NEXT: v_mov_b32_e32 v5, s1 -; SDAG-NEXT: v_mov_b32_e32 v6, s2 -; SDAG-NEXT: v_mov_b32_e32 v7, s3 -; SDAG-NEXT: v_mov_b32_e32 v17, s16 +; SDAG-NEXT: v_mov_b32_e32 v6, s0 +; SDAG-NEXT: v_mov_b32_e32 v7, s1 +; SDAG-NEXT: v_mov_b32_e32 v8, s2 +; SDAG-NEXT: v_mov_b32_e32 v9, s3 +; SDAG-NEXT: v_mov_b32_e32 v1, s16 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2 +; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2 ; SDAG-NEXT: s_nop 7 -; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] +; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__vgpr: @@ -1598,24 +1598,24 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace ; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1] +; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1] ; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54 ; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11] -; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17] ; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19] -; GISEL-NEXT: v_mov_b32_e32 v16, s2 +; GISEL-NEXT: v_mov_b32_e32 v12, s2 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_nop 0 -; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2 +; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2 ; GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GISEL-NEXT: s_nop 6 -; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1] +; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1] ; GISEL-NEXT: s_endpgm bb: %id = call i32 @llvm.amdgcn.workitem.id.x() @@ -1675,25 +1675,25 @@ define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8__sgpr(<4 x i32> inreg %arg ; SDAG-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__sgpr: ; SDAG: ; %bb.0: ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v8, s0 -; SDAG-NEXT: v_mov_b32_e32 v9, s1 -; SDAG-NEXT: v_mov_b32_e32 v10, s2 -; SDAG-NEXT: v_mov_b32_e32 v11, s3 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: v_mov_b32_e32 v4, s20 -; SDAG-NEXT: v_mov_b32_e32 v5, s21 -; SDAG-NEXT: v_mov_b32_e32 v6, s22 -; SDAG-NEXT: v_mov_b32_e32 v7, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s0 +; SDAG-NEXT: v_mov_b32_e32 v11, s1 +; SDAG-NEXT: v_mov_b32_e32 v12, s2 +; SDAG-NEXT: v_mov_b32_e32 v13, s3 +; SDAG-NEXT: v_mov_b32_e32 v2, s16 +; SDAG-NEXT: v_mov_b32_e32 v3, s17 +; SDAG-NEXT: v_mov_b32_e32 v4, s18 +; SDAG-NEXT: v_mov_b32_e32 v5, s19 +; SDAG-NEXT: v_mov_b32_e32 v6, s20 +; SDAG-NEXT: v_mov_b32_e32 v7, s21 +; SDAG-NEXT: v_mov_b32_e32 v8, s22 +; SDAG-NEXT: v_mov_b32_e32 v9, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s24 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s25 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s26 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s27 -; SDAG-NEXT: v_mov_b32_e32 v12, s28 +; SDAG-NEXT: v_mov_b32_e32 v0, s28 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 a[0:3], v[8:11], v[0:7], v12 +; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 a[0:3], v[10:13], v[2:9], v0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: v_accvgpr_read_b32 v0, a0 ; SDAG-NEXT: v_accvgpr_read_b32 v1, a1 @@ -1735,30 +1735,30 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace ; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: global_load_dwordx4 v[8:11], v0, s[6:7] +; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7] ; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 -; SDAG-NEXT: v_mov_b32_e32 v12, s8 -; SDAG-NEXT: v_mov_b32_e32 v13, s9 -; SDAG-NEXT: v_mov_b32_e32 v14, s10 -; SDAG-NEXT: v_mov_b32_e32 v15, s11 -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 +; SDAG-NEXT: v_mov_b32_e32 v14, s8 +; SDAG-NEXT: v_mov_b32_e32 v15, s9 +; SDAG-NEXT: v_mov_b32_e32 v16, s10 +; SDAG-NEXT: v_mov_b32_e32 v17, s11 +; SDAG-NEXT: v_mov_b32_e32 v2, s12 +; SDAG-NEXT: v_mov_b32_e32 v3, s13 +; SDAG-NEXT: v_mov_b32_e32 v4, s14 +; SDAG-NEXT: v_mov_b32_e32 v5, s15 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v4, s0 -; SDAG-NEXT: v_mov_b32_e32 v5, s1 -; SDAG-NEXT: v_mov_b32_e32 v6, s2 -; SDAG-NEXT: v_mov_b32_e32 v7, s3 -; SDAG-NEXT: v_mov_b32_e32 v17, s16 +; SDAG-NEXT: v_mov_b32_e32 v6, s0 +; SDAG-NEXT: v_mov_b32_e32 v7, s1 +; SDAG-NEXT: v_mov_b32_e32 v8, s2 +; SDAG-NEXT: v_mov_b32_e32 v9, s3 +; SDAG-NEXT: v_mov_b32_e32 v1, s16 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2 +; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2 ; SDAG-NEXT: s_nop 7 -; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] +; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__vgpr: @@ -1767,24 +1767,24 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace ; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1] +; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1] ; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54 ; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11] -; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17] ; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19] -; GISEL-NEXT: v_mov_b32_e32 v16, s2 +; GISEL-NEXT: v_mov_b32_e32 v12, s2 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_nop 0 -; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2 +; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2 ; GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GISEL-NEXT: s_nop 6 -; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1] +; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1] ; GISEL-NEXT: s_endpgm bb: %id = call i32 @llvm.amdgcn.workitem.id.x() @@ -1844,25 +1844,25 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8__sgpr(<4 x i32> inreg %arg ; SDAG-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__sgpr: ; SDAG: ; %bb.0: ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v8, s0 -; SDAG-NEXT: v_mov_b32_e32 v9, s1 -; SDAG-NEXT: v_mov_b32_e32 v10, s2 -; SDAG-NEXT: v_mov_b32_e32 v11, s3 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: v_mov_b32_e32 v4, s20 -; SDAG-NEXT: v_mov_b32_e32 v5, s21 -; SDAG-NEXT: v_mov_b32_e32 v6, s22 -; SDAG-NEXT: v_mov_b32_e32 v7, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s0 +; SDAG-NEXT: v_mov_b32_e32 v11, s1 +; SDAG-NEXT: v_mov_b32_e32 v12, s2 +; SDAG-NEXT: v_mov_b32_e32 v13, s3 +; SDAG-NEXT: v_mov_b32_e32 v2, s16 +; SDAG-NEXT: v_mov_b32_e32 v3, s17 +; SDAG-NEXT: v_mov_b32_e32 v4, s18 +; SDAG-NEXT: v_mov_b32_e32 v5, s19 +; SDAG-NEXT: v_mov_b32_e32 v6, s20 +; SDAG-NEXT: v_mov_b32_e32 v7, s21 +; SDAG-NEXT: v_mov_b32_e32 v8, s22 +; SDAG-NEXT: v_mov_b32_e32 v9, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s24 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s25 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s26 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s27 -; SDAG-NEXT: v_mov_b32_e32 v12, s28 +; SDAG-NEXT: v_mov_b32_e32 v0, s28 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 a[0:3], v[8:11], v[0:7], v12 +; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 a[0:3], v[10:13], v[2:9], v0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: v_accvgpr_read_b32 v0, a0 ; SDAG-NEXT: v_accvgpr_read_b32 v1, a1 @@ -1904,30 +1904,30 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace ; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: global_load_dwordx4 v[8:11], v0, s[6:7] +; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7] ; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 -; SDAG-NEXT: v_mov_b32_e32 v12, s8 -; SDAG-NEXT: v_mov_b32_e32 v13, s9 -; SDAG-NEXT: v_mov_b32_e32 v14, s10 -; SDAG-NEXT: v_mov_b32_e32 v15, s11 -; SDAG-NEXT: v_mov_b32_e32 v0, s12 -; SDAG-NEXT: v_mov_b32_e32 v1, s13 -; SDAG-NEXT: v_mov_b32_e32 v2, s14 -; SDAG-NEXT: v_mov_b32_e32 v3, s15 +; SDAG-NEXT: v_mov_b32_e32 v0, 0 +; SDAG-NEXT: v_mov_b32_e32 v14, s8 +; SDAG-NEXT: v_mov_b32_e32 v15, s9 +; SDAG-NEXT: v_mov_b32_e32 v16, s10 +; SDAG-NEXT: v_mov_b32_e32 v17, s11 +; SDAG-NEXT: v_mov_b32_e32 v2, s12 +; SDAG-NEXT: v_mov_b32_e32 v3, s13 +; SDAG-NEXT: v_mov_b32_e32 v4, s14 +; SDAG-NEXT: v_mov_b32_e32 v5, s15 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v4, s0 -; SDAG-NEXT: v_mov_b32_e32 v5, s1 -; SDAG-NEXT: v_mov_b32_e32 v6, s2 -; SDAG-NEXT: v_mov_b32_e32 v7, s3 -; SDAG-NEXT: v_mov_b32_e32 v17, s16 +; SDAG-NEXT: v_mov_b32_e32 v6, s0 +; SDAG-NEXT: v_mov_b32_e32 v7, s1 +; SDAG-NEXT: v_mov_b32_e32 v8, s2 +; SDAG-NEXT: v_mov_b32_e32 v9, s3 +; SDAG-NEXT: v_mov_b32_e32 v1, s16 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2 +; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2 ; SDAG-NEXT: s_nop 7 -; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] +; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__vgpr: @@ -1936,24 +1936,24 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace ; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1] +; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1] ; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54 ; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11] -; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17] ; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19] -; GISEL-NEXT: v_mov_b32_e32 v16, s2 +; GISEL-NEXT: v_mov_b32_e32 v12, s2 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_nop 0 -; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2 +; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2 ; GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GISEL-NEXT: s_nop 6 -; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1] +; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1] ; GISEL-NEXT: s_endpgm bb: %id = call i32 @llvm.amdgcn.workitem.id.x() @@ -2013,25 +2013,25 @@ define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8__sgpr(<4 x i32> inreg %arg ; SDAG-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__sgpr: ; SDAG: ; %bb.0: ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v8, s0 -; SDAG-NEXT: v_mov_b32_e32 v9, s1 -; SDAG-NEXT: v_mov_b32_e32 v10, s2 -; SDAG-NEXT: v_mov_b32_e32 v11, s3 -; SDAG-NEXT: v_mov_b32_e32 v0, s16 -; SDAG-NEXT: v_mov_b32_e32 v1, s17 -; SDAG-NEXT: v_mov_b32_e32 v2, s18 -; SDAG-NEXT: v_mov_b32_e32 v3, s19 -; SDAG-NEXT: v_mov_b32_e32 v4, s20 -; SDAG-NEXT: v_mov_b32_e32 v5, s21 -; SDAG-NEXT: v_mov_b32_e32 v6, s22 -; SDAG-NEXT: v_mov_b32_e32 v7, s23 +; SDAG-NEXT: v_mov_b32_e32 v10, s0 +; SDAG-NEXT: v_mov_b32_e32 v11, s1 +; SDAG-NEXT: v_mov_b32_e32 v12, s2 +; SDAG-NEXT: v_mov_b32_e32 v13, s3 +; SDAG-NEXT: v_mov_b32_e32 v2, s16 +; SDAG-NEXT: v_mov_b32_e32 v3, s17 +; SDAG-NEXT: v_mov_b32_e32 v4, s18 +; SDAG-NEXT: v_mov_b32_e32 v5, s19 +; SDAG-NEXT: v_mov_b32_e32 v6, s20 +; SDAG-NEXT: v_mov_b32_e32 v7, s21 +; SDAG-NEXT: v_mov_b32_e32 v8, s22 +; SDAG-NEXT: v_mov_b32_e32 v9, s23 ; SDAG-NEXT: v_accvgpr_write_b32 a0, s24 ; SDAG-NEXT: v_accvgpr_write_b32 a1, s25 ; SDAG-NEXT: v_accvgpr_write_b32 a2, s26 ; SDAG-NEXT: v_accvgpr_write_b32 a3, s27 -; SDAG-NEXT: v_mov_b32_e32 v12, s28 +; SDAG-NEXT: v_mov_b32_e32 v0, s28 ; SDAG-NEXT: s_nop 1 -; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 a[0:3], v[8:11], v[0:7], v12 +; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 a[0:3], v[10:13], v[2:9], v0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: v_accvgpr_read_b32 v0, a0 ; SDAG-NEXT: v_accvgpr_read_b32 v1, a1 @@ -2081,22 +2081,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace( ; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v24, s8 -; SDAG-NEXT: v_mov_b32_e32 v25, s9 -; SDAG-NEXT: v_mov_b32_e32 v26, s10 -; SDAG-NEXT: v_mov_b32_e32 v27, s11 -; SDAG-NEXT: v_mov_b32_e32 v16, s12 -; SDAG-NEXT: v_mov_b32_e32 v17, s13 -; SDAG-NEXT: v_mov_b32_e32 v18, s14 -; SDAG-NEXT: v_mov_b32_e32 v19, s15 -; SDAG-NEXT: v_mov_b32_e32 v20, s0 -; SDAG-NEXT: v_mov_b32_e32 v21, s1 -; SDAG-NEXT: v_mov_b32_e32 v22, s2 -; SDAG-NEXT: v_mov_b32_e32 v23, s3 -; SDAG-NEXT: v_mov_b32_e32 v28, s16 +; SDAG-NEXT: v_mov_b32_e32 v26, s8 +; SDAG-NEXT: v_mov_b32_e32 v27, s9 +; SDAG-NEXT: v_mov_b32_e32 v28, s10 +; SDAG-NEXT: v_mov_b32_e32 v29, s11 +; SDAG-NEXT: v_mov_b32_e32 v18, s12 +; SDAG-NEXT: v_mov_b32_e32 v19, s13 +; SDAG-NEXT: v_mov_b32_e32 v20, s14 +; SDAG-NEXT: v_mov_b32_e32 v21, s15 +; SDAG-NEXT: v_mov_b32_e32 v22, s0 +; SDAG-NEXT: v_mov_b32_e32 v23, s1 +; SDAG-NEXT: v_mov_b32_e32 v24, s2 +; SDAG-NEXT: v_mov_b32_e32 v25, s3 +; SDAG-NEXT: v_mov_b32_e32 v16, s16 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2 +; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2 ; SDAG-NEXT: v_mov_b32_e32 v16, 0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 2 @@ -2454,22 +2454,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace( ; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v24, s8 -; SDAG-NEXT: v_mov_b32_e32 v25, s9 -; SDAG-NEXT: v_mov_b32_e32 v26, s10 -; SDAG-NEXT: v_mov_b32_e32 v27, s11 -; SDAG-NEXT: v_mov_b32_e32 v16, s12 -; SDAG-NEXT: v_mov_b32_e32 v17, s13 -; SDAG-NEXT: v_mov_b32_e32 v18, s14 -; SDAG-NEXT: v_mov_b32_e32 v19, s15 -; SDAG-NEXT: v_mov_b32_e32 v20, s0 -; SDAG-NEXT: v_mov_b32_e32 v21, s1 -; SDAG-NEXT: v_mov_b32_e32 v22, s2 -; SDAG-NEXT: v_mov_b32_e32 v23, s3 -; SDAG-NEXT: v_mov_b32_e32 v28, s16 +; SDAG-NEXT: v_mov_b32_e32 v26, s8 +; SDAG-NEXT: v_mov_b32_e32 v27, s9 +; SDAG-NEXT: v_mov_b32_e32 v28, s10 +; SDAG-NEXT: v_mov_b32_e32 v29, s11 +; SDAG-NEXT: v_mov_b32_e32 v18, s12 +; SDAG-NEXT: v_mov_b32_e32 v19, s13 +; SDAG-NEXT: v_mov_b32_e32 v20, s14 +; SDAG-NEXT: v_mov_b32_e32 v21, s15 +; SDAG-NEXT: v_mov_b32_e32 v22, s0 +; SDAG-NEXT: v_mov_b32_e32 v23, s1 +; SDAG-NEXT: v_mov_b32_e32 v24, s2 +; SDAG-NEXT: v_mov_b32_e32 v25, s3 +; SDAG-NEXT: v_mov_b32_e32 v16, s16 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2 +; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2 ; SDAG-NEXT: v_mov_b32_e32 v16, 0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 2 @@ -2827,22 +2827,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace( ; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v24, s8 -; SDAG-NEXT: v_mov_b32_e32 v25, s9 -; SDAG-NEXT: v_mov_b32_e32 v26, s10 -; SDAG-NEXT: v_mov_b32_e32 v27, s11 -; SDAG-NEXT: v_mov_b32_e32 v16, s12 -; SDAG-NEXT: v_mov_b32_e32 v17, s13 -; SDAG-NEXT: v_mov_b32_e32 v18, s14 -; SDAG-NEXT: v_mov_b32_e32 v19, s15 -; SDAG-NEXT: v_mov_b32_e32 v20, s0 -; SDAG-NEXT: v_mov_b32_e32 v21, s1 -; SDAG-NEXT: v_mov_b32_e32 v22, s2 -; SDAG-NEXT: v_mov_b32_e32 v23, s3 -; SDAG-NEXT: v_mov_b32_e32 v28, s16 +; SDAG-NEXT: v_mov_b32_e32 v26, s8 +; SDAG-NEXT: v_mov_b32_e32 v27, s9 +; SDAG-NEXT: v_mov_b32_e32 v28, s10 +; SDAG-NEXT: v_mov_b32_e32 v29, s11 +; SDAG-NEXT: v_mov_b32_e32 v18, s12 +; SDAG-NEXT: v_mov_b32_e32 v19, s13 +; SDAG-NEXT: v_mov_b32_e32 v20, s14 +; SDAG-NEXT: v_mov_b32_e32 v21, s15 +; SDAG-NEXT: v_mov_b32_e32 v22, s0 +; SDAG-NEXT: v_mov_b32_e32 v23, s1 +; SDAG-NEXT: v_mov_b32_e32 v24, s2 +; SDAG-NEXT: v_mov_b32_e32 v25, s3 +; SDAG-NEXT: v_mov_b32_e32 v16, s16 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2 +; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2 ; SDAG-NEXT: v_mov_b32_e32 v16, 0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 2 @@ -3200,22 +3200,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace( ; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64 ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mov_b32_e32 v24, s8 -; SDAG-NEXT: v_mov_b32_e32 v25, s9 -; SDAG-NEXT: v_mov_b32_e32 v26, s10 -; SDAG-NEXT: v_mov_b32_e32 v27, s11 -; SDAG-NEXT: v_mov_b32_e32 v16, s12 -; SDAG-NEXT: v_mov_b32_e32 v17, s13 -; SDAG-NEXT: v_mov_b32_e32 v18, s14 -; SDAG-NEXT: v_mov_b32_e32 v19, s15 -; SDAG-NEXT: v_mov_b32_e32 v20, s0 -; SDAG-NEXT: v_mov_b32_e32 v21, s1 -; SDAG-NEXT: v_mov_b32_e32 v22, s2 -; SDAG-NEXT: v_mov_b32_e32 v23, s3 -; SDAG-NEXT: v_mov_b32_e32 v28, s16 +; SDAG-NEXT: v_mov_b32_e32 v26, s8 +; SDAG-NEXT: v_mov_b32_e32 v27, s9 +; SDAG-NEXT: v_mov_b32_e32 v28, s10 +; SDAG-NEXT: v_mov_b32_e32 v29, s11 +; SDAG-NEXT: v_mov_b32_e32 v18, s12 +; SDAG-NEXT: v_mov_b32_e32 v19, s13 +; SDAG-NEXT: v_mov_b32_e32 v20, s14 +; SDAG-NEXT: v_mov_b32_e32 v21, s15 +; SDAG-NEXT: v_mov_b32_e32 v22, s0 +; SDAG-NEXT: v_mov_b32_e32 v23, s1 +; SDAG-NEXT: v_mov_b32_e32 v24, s2 +; SDAG-NEXT: v_mov_b32_e32 v25, s3 +; SDAG-NEXT: v_mov_b32_e32 v16, s16 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: s_nop 0 -; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2 +; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2 ; SDAG-NEXT: v_mov_b32_e32 v16, 0 ; SDAG-NEXT: s_nop 7 ; SDAG-NEXT: s_nop 2 diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll index 5b2de59..84123e6 100644 --- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll @@ -121,18 +121,7 @@ define <2 x bfloat> @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_undeflo_clamp_postcvt ; GFX1250: ; %bb.0: ; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-NEXT: s_wait_kmcnt 0x0 -; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1250-NEXT: v_max_num_f32_e32 v0, 0, v0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 -; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_min_num_f32_e32 v0, 1.0, v0 -; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-NEXT: v_fma_mixhi_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp ; GFX1250-NEXT: s_set_pc_i64 s[30:31] %src0.ext = fpext bfloat %src0 to float %src1.ext = fpext bfloat %src1 to float @@ -150,20 +139,10 @@ define <2 x bfloat> @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_undeflo_clamp_postcvt ; GFX1250: ; %bb.0: ; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-NEXT: s_wait_kmcnt 0x0 -; GFX1250-NEXT: v_fma_mixlo_bf16 v1, v0, v1, v2 op_sel_hi:[1,1,1] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v1 -; GFX1250-NEXT: v_max_num_f32_e32 v0, 0, v0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 -; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_min_num_f32_e32 v0, 1.0, v0 -; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v3, v0, v1, v2 op_sel_hi:[1,1,1] +; GFX1250-NEXT: v_fma_mixhi_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp ; GFX1250-NEXT: s_wait_storecnt 0x0 -; GFX1250-NEXT: global_store_b16 v[0:1], v1, off scope:SCOPE_SYS +; GFX1250-NEXT: global_store_b16 v[0:1], v3, off scope:SCOPE_SYS ; GFX1250-NEXT: s_wait_storecnt 0x0 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] %src0.ext = fpext bfloat %src0 to float diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll index 557080a..1b2eb83 100644 --- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll @@ -75,15 +75,8 @@ define bfloat @v_mad_mixlo_bf16_bf16lo_bf16lo_f32_clamp_post_cvt(bfloat %src0, b ; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,0] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1250-NEXT: v_max_num_f32_e32 v0, 0, v0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 -; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_min_num_f32_e32 v0, 1.0, v0 -; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp ; GFX1250-NEXT: s_set_pc_i64 s[30:31] %src0.ext = fpext bfloat %src0 to float %src1.ext = fpext bfloat %src1 to float @@ -199,9 +192,8 @@ define <2 x bfloat> @v_mad_mix_v2f32_clamp_postcvt(<2 x bfloat> %src0, <2 x bflo ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[4:5], v[6:7], v[0:1] ; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, 0 -; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, 1.0 op_sel_hi:[1,0] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp ; GFX1250-NEXT: s_set_pc_i64 s[30:31] %src0.ext = fpext <2 x bfloat> %src0 to <2 x float> %src1.ext = fpext <2 x bfloat> %src1 to <2 x float> @@ -219,16 +211,13 @@ define <3 x bfloat> @v_mad_mix_v3f32_clamp_postcvt(<3 x bfloat> %src0, <3 x bflo ; GFX1250: ; %bb.0: ; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-NEXT: s_wait_kmcnt 0x0 -; GFX1250-NEXT: v_fma_mixlo_bf16 v6, v0, v2, v4 op_sel_hi:[1,1,1] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1250-NEXT: v_fma_mixhi_bf16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] -; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v1, v3, v5 op_sel_hi:[1,1,1] -; GFX1250-NEXT: v_pk_max_num_bf16 v1, v6, 0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp +; GFX1250-NEXT: v_fma_mixlo_bf16 v1, v1, v3, v5 op_sel_hi:[1,1,1] ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-NEXT: v_pk_max_num_bf16 v2, v0, 0 -; GFX1250-NEXT: v_pk_min_num_bf16 v0, v1, 1.0 op_sel_hi:[1,0] +; GFX1250-NEXT: v_fma_mixhi_bf16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp +; GFX1250-NEXT: v_pk_max_num_bf16 v1, v1, v1 clamp ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1250-NEXT: v_pk_min_num_bf16 v1, v2, 1.0 +; GFX1250-NEXT: v_mov_b32_e32 v0, v6 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] %src0.ext = fpext <3 x bfloat> %src0 to <3 x float> %src1.ext = fpext <3 x bfloat> %src1 to <3 x float> @@ -261,11 +250,8 @@ define <4 x bfloat> @v_mad_mix_v4f32_clamp_postcvt(<4 x bfloat> %src0, <4 x bflo ; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 ; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v2, v3 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, 0 -; GFX1250-NEXT: v_pk_max_num_bf16 v1, v1, 0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, 1.0 op_sel_hi:[1,0] -; GFX1250-NEXT: v_pk_min_num_bf16 v1, v1, 1.0 op_sel_hi:[1,0] +; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp +; GFX1250-NEXT: v_pk_max_num_bf16 v1, v1, v1 clamp ; GFX1250-NEXT: s_set_pc_i64 s[30:31] %src0.ext = fpext <4 x bfloat> %src0 to <4 x float> %src1.ext = fpext <4 x bfloat> %src1 to <4 x float> @@ -291,15 +277,7 @@ define <2 x bfloat> @v_mad_mix_v2f32_clamp_postcvt_lo(<2 x bfloat> %src0, <2 x b ; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[4:5], v[6:7], v[0:1] ; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0 -; GFX1250-NEXT: v_max_num_f32_e32 v1, 0, v1 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_min_num_f32_e32 v1, 1.0, v1 -; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_pk_max_num_bf16 v1, v0, v0 clamp ; GFX1250-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] %src0.ext = fpext <2 x bfloat> %src0 to <2 x float> @@ -328,14 +306,8 @@ define <2 x bfloat> @v_mad_mix_v2f32_clamp_postcvt_hi(<2 x bfloat> %src0, <2 x b ; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[4:5], v[6:7], v[0:1] ; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v0 -; GFX1250-NEXT: v_max_num_f32_e32 v1, 0, v1 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_min_num_f32_e32 v1, 1.0, v1 -; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 +; GFX1250-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX1250-NEXT: v_pk_max_num_bf16 v1, v1, v1 clamp ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] diff --git a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll index 9cc42ac..be02045 100644 --- a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll +++ b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll @@ -9851,8 +9851,8 @@ define void @memmove_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5 ; CHECK-NEXT: s_andn2_saveexec_b32 s6, s6 ; CHECK-NEXT: s_cbranch_execz .LBB8_6 ; CHECK-NEXT: ; %bb.4: ; %memmove_bwd_loop.preheader -; CHECK-NEXT: v_add_nc_u32_e32 v0, 0x700, v0 ; CHECK-NEXT: v_add_nc_u32_e32 v1, 0x700, v1 +; CHECK-NEXT: v_add_nc_u32_e32 v0, 0x700, v0 ; CHECK-NEXT: s_movk_i32 s4, 0xf800 ; CHECK-NEXT: s_mov_b32 s5, -1 ; CHECK-NEXT: .LBB8_5: ; %memmove_bwd_loop @@ -11167,8 +11167,8 @@ define void @memmove_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5 ; ALIGNED-NEXT: s_andn2_saveexec_b32 s6, s6 ; ALIGNED-NEXT: s_cbranch_execz .LBB8_6 ; ALIGNED-NEXT: ; %bb.4: ; %memmove_bwd_loop.preheader -; ALIGNED-NEXT: v_add_nc_u32_e32 v0, 0x700, v0 ; ALIGNED-NEXT: v_add_nc_u32_e32 v1, 0x700, v1 +; ALIGNED-NEXT: v_add_nc_u32_e32 v0, 0x700, v0 ; ALIGNED-NEXT: s_movk_i32 s4, 0xf800 ; ALIGNED-NEXT: s_mov_b32 s5, -1 ; ALIGNED-NEXT: .LBB8_5: ; %memmove_bwd_loop @@ -12381,8 +12381,8 @@ define void @memmove_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5 ; UNROLL3-NEXT: buffer_load_dword v4, v1, s[0:3], 0 offen offset:2024 ; UNROLL3-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen offset:2020 ; UNROLL3-NEXT: buffer_load_dword v6, v1, s[0:3], 0 offen offset:2016 -; UNROLL3-NEXT: v_add_nc_u32_e32 v2, 0x7b0, v0 ; UNROLL3-NEXT: v_add_nc_u32_e32 v1, 0x7b0, v1 +; UNROLL3-NEXT: v_add_nc_u32_e32 v2, 0x7b0, v0 ; UNROLL3-NEXT: s_waitcnt vmcnt(3) ; UNROLL3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:2028 ; UNROLL3-NEXT: s_waitcnt vmcnt(2) diff --git a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll index 6763957..f7aaa3e 100644 --- a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll +++ b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll @@ -1,15 +1,148 @@ -; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --enable-var-scope --check-prefixes=GCN,GFX908 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN,GFX90A %s -; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN,GFX90A %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --enable-var-scope --check-prefixes=GCN,GFX90A %s -; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --enable-var-scope --check-prefixes=GCN,GFX90A %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX90A %s +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX90A %s declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32) -; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_vgpr: -; GFX908: v_mfma_f32_32x32x1{{.*}} a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, a[{{[0-9:]+}}] -; GFX90A: v_mfma_f32_32x32x1{{.*}} v[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, v[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg) #0 { +; GFX908-LABEL: test_mfma_f32_32x32x1f32_vgpr: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX908-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v0, s16 +; GFX908-NEXT: v_mov_b32_e32 v1, s17 +; GFX908-NEXT: v_mov_b32_e32 v2, s18 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s21 +; GFX908-NEXT: v_mov_b32_e32 v1, s22 +; GFX908-NEXT: v_mov_b32_e32 v2, s23 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s24 +; GFX908-NEXT: v_mov_b32_e32 v1, s25 +; GFX908-NEXT: v_mov_b32_e32 v2, s26 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s27 +; GFX908-NEXT: v_mov_b32_e32 v1, s28 +; GFX908-NEXT: v_mov_b32_e32 v2, s29 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s30 +; GFX908-NEXT: v_mov_b32_e32 v1, s31 +; GFX908-NEXT: v_mov_b32_e32 v2, s0 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s1 +; GFX908-NEXT: v_mov_b32_e32 v1, s2 +; GFX908-NEXT: v_mov_b32_e32 v2, s3 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s4 +; GFX908-NEXT: v_mov_b32_e32 v1, s5 +; GFX908-NEXT: v_mov_b32_e32 v2, s6 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s7 +; GFX908-NEXT: v_mov_b32_e32 v1, s8 +; GFX908-NEXT: v_mov_b32_e32 v2, s9 +; GFX908-NEXT: v_mov_b32_e32 v3, s19 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s10 +; GFX908-NEXT: v_mov_b32_e32 v1, s11 +; GFX908-NEXT: v_mov_b32_e32 v2, s12 +; GFX908-NEXT: v_mov_b32_e32 v5, s20 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s13 +; GFX908-NEXT: v_mov_b32_e32 v1, s14 +; GFX908-NEXT: v_mov_b32_e32 v2, s15 +; GFX908-NEXT: v_mov_b32_e32 v3, 1.0 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a31, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, 2.0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a28 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a16 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a20 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a8 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a12 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; GFX908-NEXT: s_endpgm bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0) @@ -17,9 +150,142 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_agpr: -; GCN: v_mfma_f32_32x32x1{{.*}} a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg) #2 { +; GFX908-LABEL: test_mfma_f32_32x32x1f32_agpr: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX908-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v0, s16 +; GFX908-NEXT: v_mov_b32_e32 v1, s17 +; GFX908-NEXT: v_mov_b32_e32 v2, s18 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s21 +; GFX908-NEXT: v_mov_b32_e32 v1, s22 +; GFX908-NEXT: v_mov_b32_e32 v2, s23 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s24 +; GFX908-NEXT: v_mov_b32_e32 v1, s25 +; GFX908-NEXT: v_mov_b32_e32 v2, s26 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s27 +; GFX908-NEXT: v_mov_b32_e32 v1, s28 +; GFX908-NEXT: v_mov_b32_e32 v2, s29 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s30 +; GFX908-NEXT: v_mov_b32_e32 v1, s31 +; GFX908-NEXT: v_mov_b32_e32 v2, s0 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s1 +; GFX908-NEXT: v_mov_b32_e32 v1, s2 +; GFX908-NEXT: v_mov_b32_e32 v2, s3 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s4 +; GFX908-NEXT: v_mov_b32_e32 v1, s5 +; GFX908-NEXT: v_mov_b32_e32 v2, s6 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s7 +; GFX908-NEXT: v_mov_b32_e32 v1, s8 +; GFX908-NEXT: v_mov_b32_e32 v2, s9 +; GFX908-NEXT: v_mov_b32_e32 v3, s19 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s10 +; GFX908-NEXT: v_mov_b32_e32 v1, s11 +; GFX908-NEXT: v_mov_b32_e32 v2, s12 +; GFX908-NEXT: v_mov_b32_e32 v5, s20 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, s13 +; GFX908-NEXT: v_mov_b32_e32 v1, s14 +; GFX908-NEXT: v_mov_b32_e32 v2, s15 +; GFX908-NEXT: v_mov_b32_e32 v3, 1.0 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a31, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, 2.0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a28 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a16 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a20 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a8 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a12 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; GFX908-NEXT: s_endpgm bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0) @@ -27,9 +293,105 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_inline_asm_virtual_agpr: -; GCN: v_mfma_f32_32x32x1{{.*}} a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_virtual_agpr(ptr addrspace(1) %arg) { +; GFX908-LABEL: test_mfma_f32_32x32x1f32_inline_asm_virtual_agpr: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ; def a0 +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112 +; GFX908-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96 +; GFX908-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80 +; GFX908-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64 +; GFX908-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48 +; GFX908-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32 +; GFX908-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16 +; GFX908-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1] +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a0, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v6 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v7 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v8 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v9 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v10 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v11 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v12 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v13 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v14 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v15 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v16 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v17 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v18 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v19 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v20 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v21 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v22 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v23 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v24 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v25 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v26 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v27 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v28 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v29 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v30 +; GFX908-NEXT: v_accvgpr_write_b32 a31, v31 +; GFX908-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a0 +; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 +; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112 +; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64 +; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80 +; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 +; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48 +; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] +; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16 +; GFX908-NEXT: s_endpgm bb: %acc = call i32 asm sideeffect "; def $0", "={a0}"() %in.1 = load <32 x float>, ptr addrspace(1) %arg @@ -38,9 +400,105 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_inline_asm_phys_agpr: -; GCN: v_mfma_f32_32x32x1{{.*}} a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_phys_agpr(ptr addrspace(1) %arg) { +; GFX908-LABEL: test_mfma_f32_32x32x1f32_inline_asm_phys_agpr: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ; use a[100:131] +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112 +; GFX908-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96 +; GFX908-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80 +; GFX908-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64 +; GFX908-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48 +; GFX908-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32 +; GFX908-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16 +; GFX908-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1] +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a0, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v6 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v7 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v8 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v9 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v10 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v11 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v12 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v13 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v14 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v15 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v16 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v17 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v18 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v19 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v20 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v21 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v22 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v23 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v24 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v25 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v26 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v27 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v28 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v29 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v30 +; GFX908-NEXT: v_accvgpr_write_b32 a31, v31 +; GFX908-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a0 +; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 +; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112 +; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64 +; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80 +; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 +; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48 +; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] +; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16 +; GFX908-NEXT: s_endpgm bb: call void asm sideeffect "; use $0", "{a[100:131]}"(<32 x float> poison) %in.1 = load <32 x float>, ptr addrspace(1) %arg @@ -49,10 +507,105 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_inline_asm_no_agprs: -; GFX908: v_mfma_f32_32x32x1{{.*}} a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, a[{{[0-9:]+}}] -; GFX90A: v_mfma_f32_32x32x1{{.*}} v[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, v[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_no_agprs(ptr addrspace(1) %arg) #0 { +; GFX908-LABEL: test_mfma_f32_32x32x1f32_inline_asm_no_agprs: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ; def v0 +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112 +; GFX908-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96 +; GFX908-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80 +; GFX908-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64 +; GFX908-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48 +; GFX908-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32 +; GFX908-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16 +; GFX908-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1] +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a0, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v6 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v7 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v8 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v9 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v10 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v11 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v12 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v13 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v14 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v15 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v16 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v17 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v18 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v19 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v20 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v21 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v22 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v23 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v24 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v25 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v26 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v27 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v28 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v29 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v30 +; GFX908-NEXT: v_accvgpr_write_b32 a31, v31 +; GFX908-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a0 +; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 +; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112 +; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64 +; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80 +; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 +; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48 +; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] +; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16 +; GFX908-NEXT: s_endpgm bb: %acc = call i32 asm sideeffect "; def $0", "={v0}"() %in.1 = load <32 x float>, ptr addrspace(1) %arg @@ -61,9 +614,127 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_call: -; GCN: v_mfma_f32_32x32x1{{.*}} a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call(ptr addrspace(1) %arg) #1 { +; GFX908-LABEL: test_mfma_f32_32x32x1f32_call: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX908-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX908-NEXT: s_mov_b32 s38, -1 +; GFX908-NEXT: s_mov_b32 s39, 0xe00000 +; GFX908-NEXT: s_add_u32 s36, s36, s11 +; GFX908-NEXT: s_addc_u32 s37, s37, 0 +; GFX908-NEXT: s_mov_b32 s12, s8 +; GFX908-NEXT: s_add_u32 s8, s4, 44 +; GFX908-NEXT: s_mov_b32 s13, s9 +; GFX908-NEXT: s_addc_u32 s9, s5, 0 +; GFX908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX908-NEXT: s_getpc_b64 s[4:5] +; GFX908-NEXT: s_add_u32 s4, s4, foo@gotpcrel32@lo+4 +; GFX908-NEXT: s_addc_u32 s5, s5, foo@gotpcrel32@hi+12 +; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX908-NEXT: s_mov_b32 s14, s10 +; GFX908-NEXT: s_mov_b64 s[10:11], s[6:7] +; GFX908-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX908-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX908-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX908-NEXT: s_mov_b64 s[6:7], s[2:3] +; GFX908-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX908-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX908-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX908-NEXT: s_mov_b32 s32, 0 +; GFX908-NEXT: v_mov_b32_e32 v40, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX908-NEXT: global_load_dwordx4 v[28:31], v40, s[34:35] offset:112 +; GFX908-NEXT: global_load_dwordx4 v[24:27], v40, s[34:35] offset:96 +; GFX908-NEXT: global_load_dwordx4 v[20:23], v40, s[34:35] offset:80 +; GFX908-NEXT: global_load_dwordx4 v[16:19], v40, s[34:35] offset:64 +; GFX908-NEXT: global_load_dwordx4 v[12:15], v40, s[34:35] offset:48 +; GFX908-NEXT: global_load_dwordx4 v[8:11], v40, s[34:35] offset:32 +; GFX908-NEXT: global_load_dwordx4 v[4:7], v40, s[34:35] offset:16 +; GFX908-NEXT: global_load_dwordx4 v[0:3], v40, s[34:35] +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a0, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v6 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v7 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v8 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v9 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v10 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v11 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v12 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v13 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v14 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v15 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v16 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v17 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v18 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v19 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v20 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v21 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v22 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v23 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v24 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v25 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v26 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v27 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v28 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v29 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v30 +; GFX908-NEXT: v_accvgpr_write_b32 a31, v31 +; GFX908-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a0 +; GFX908-NEXT: global_store_dwordx4 v40, v[0:3], s[34:35] offset:96 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 +; GFX908-NEXT: global_store_dwordx4 v40, v[4:7], s[34:35] offset:112 +; GFX908-NEXT: global_store_dwordx4 v40, v[8:11], s[34:35] offset:64 +; GFX908-NEXT: global_store_dwordx4 v40, v[12:15], s[34:35] offset:80 +; GFX908-NEXT: global_store_dwordx4 v40, v[16:19], s[34:35] offset:32 +; GFX908-NEXT: global_store_dwordx4 v40, v[20:23], s[34:35] offset:48 +; GFX908-NEXT: global_store_dwordx4 v40, v[24:27], s[34:35] +; GFX908-NEXT: global_store_dwordx4 v40, v[0:3], s[34:35] offset:16 +; GFX908-NEXT: s_endpgm bb: call void @foo() %in.1 = load <32 x float>, ptr addrspace(1) %arg @@ -75,10 +746,173 @@ bb: ; We could avoid scan to find calls since we see these during lowering before selection. ; However, in SDag lowering and selection is done block by block, so it would only work ; in Global ISel. - -; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_call_multi_bb: -; GCN: v_mfma_f32_32x32x1{{.*}} a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call_multi_bb(ptr addrspace(1) %arg, i1 %c0) #1 { +; GFX908-LABEL: test_mfma_f32_32x32x1f32_call_multi_bb: +; GFX908: ; %bb.0: ; %bb1 +; GFX908-NEXT: s_mov_b32 s52, SCRATCH_RSRC_DWORD0 +; GFX908-NEXT: s_mov_b32 s53, SCRATCH_RSRC_DWORD1 +; GFX908-NEXT: s_mov_b32 s54, -1 +; GFX908-NEXT: s_mov_b32 s55, 0xe00000 +; GFX908-NEXT: s_add_u32 s52, s52, s11 +; GFX908-NEXT: s_mov_b32 s14, s10 +; GFX908-NEXT: s_mov_b32 s12, s8 +; GFX908-NEXT: s_mov_b64 s[10:11], s[6:7] +; GFX908-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX908-NEXT: s_load_dword s8, s[4:5], 0x2c +; GFX908-NEXT: v_mov_b32_e32 v6, 1.0 +; GFX908-NEXT: v_mov_b32_e32 v7, 0 +; GFX908-NEXT: s_addc_u32 s53, s53, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx16 s[36:51], s[6:7], 0x0 +; GFX908-NEXT: s_load_dwordx16 s[16:31], s[6:7], 0x40 +; GFX908-NEXT: s_bitcmp0_b32 s8, 0 +; GFX908-NEXT: s_mov_b32 s32, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v3, s36 +; GFX908-NEXT: v_mov_b32_e32 v4, s37 +; GFX908-NEXT: v_mov_b32_e32 v5, s40 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v4 +; GFX908-NEXT: v_mov_b32_e32 v3, s38 +; GFX908-NEXT: v_mov_b32_e32 v4, s39 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v4 +; GFX908-NEXT: v_mov_b32_e32 v3, s41 +; GFX908-NEXT: v_mov_b32_e32 v4, s42 +; GFX908-NEXT: v_mov_b32_e32 v5, s43 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v5 +; GFX908-NEXT: v_mov_b32_e32 v3, s44 +; GFX908-NEXT: v_mov_b32_e32 v4, s45 +; GFX908-NEXT: v_mov_b32_e32 v5, s46 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v5 +; GFX908-NEXT: v_mov_b32_e32 v3, s47 +; GFX908-NEXT: v_mov_b32_e32 v4, s48 +; GFX908-NEXT: v_mov_b32_e32 v5, s49 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v5 +; GFX908-NEXT: v_mov_b32_e32 v3, s50 +; GFX908-NEXT: v_mov_b32_e32 v4, s51 +; GFX908-NEXT: v_mov_b32_e32 v5, s16 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v5 +; GFX908-NEXT: v_mov_b32_e32 v3, s17 +; GFX908-NEXT: v_mov_b32_e32 v4, s18 +; GFX908-NEXT: v_mov_b32_e32 v5, s19 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v5 +; GFX908-NEXT: v_mov_b32_e32 v3, s20 +; GFX908-NEXT: v_mov_b32_e32 v4, s21 +; GFX908-NEXT: v_mov_b32_e32 v5, s22 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v5 +; GFX908-NEXT: v_mov_b32_e32 v3, s23 +; GFX908-NEXT: v_mov_b32_e32 v4, s24 +; GFX908-NEXT: v_mov_b32_e32 v5, s25 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v5 +; GFX908-NEXT: v_mov_b32_e32 v3, s26 +; GFX908-NEXT: v_mov_b32_e32 v4, s27 +; GFX908-NEXT: v_mov_b32_e32 v5, s28 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v5 +; GFX908-NEXT: v_mov_b32_e32 v3, s29 +; GFX908-NEXT: v_mov_b32_e32 v4, s30 +; GFX908-NEXT: v_mov_b32_e32 v5, s31 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a31, v5 +; GFX908-NEXT: v_mov_b32_e32 v3, 2.0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v6, v3, a[0:31] cbsz:1 abid:2 blgp:3 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a24 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:96 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a28 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:112 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a16 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:64 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a20 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:80 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a8 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:32 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a12 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:48 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a4 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:16 +; GFX908-NEXT: s_cbranch_scc1 .LBB6_2 +; GFX908-NEXT: ; %bb.1: ; %bb2 +; GFX908-NEXT: s_add_u32 s8, s4, 48 +; GFX908-NEXT: s_mov_b32 s13, s9 +; GFX908-NEXT: s_addc_u32 s9, s5, 0 +; GFX908-NEXT: s_getpc_b64 s[4:5] +; GFX908-NEXT: s_add_u32 s4, s4, foo@gotpcrel32@lo+4 +; GFX908-NEXT: s_addc_u32 s5, s5, foo@gotpcrel32@hi+12 +; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX908-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX908-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX908-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX908-NEXT: s_mov_b64 s[6:7], s[2:3] +; GFX908-NEXT: s_mov_b64 s[0:1], s[52:53] +; GFX908-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX908-NEXT: s_mov_b64 s[2:3], s[54:55] +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX908-NEXT: .LBB6_2: ; %bb3 +; GFX908-NEXT: s_endpgm bb1: %in.1 = load <32 x float>, ptr addrspace(1) %arg %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 1, i32 2, i32 3) @@ -94,10 +928,101 @@ bb3: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_nonentry_noagpr: -; GFX908: v_mfma_f32_32x32x1{{.*}} a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, a[{{[0-9:]+}}] -; GFX90A: v_mfma_f32_32x32x1{{.*}} v[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, v[{{[0-9:]+}}] define void @test_mfma_f32_32x32x1f32_nonentry_noagpr(ptr addrspace(1) %arg) #0 { +; GFX908-LABEL: test_mfma_f32_32x32x1f32_nonentry_noagpr: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX908-NEXT: global_load_dwordx4 v[30:33], v[0:1], off offset:112 +; GFX908-NEXT: global_load_dwordx4 v[26:29], v[0:1], off offset:96 +; GFX908-NEXT: global_load_dwordx4 v[22:25], v[0:1], off offset:80 +; GFX908-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:64 +; GFX908-NEXT: global_load_dwordx4 v[14:17], v[0:1], off offset:48 +; GFX908-NEXT: global_load_dwordx4 v[10:13], v[0:1], off offset:32 +; GFX908-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:16 +; GFX908-NEXT: global_load_dwordx4 v[2:5], v[0:1], off +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a0, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v6 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v7 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v8 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v9 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v10 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v11 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v12 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v13 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v14 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v15 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v16 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v18 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v19 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v20 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v21 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v22 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v23 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v24 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v25 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v26 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v27 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v28 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v29 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v30 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v31 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v32 +; GFX908-NEXT: v_accvgpr_write_b32 a31, v33 +; GFX908-NEXT: v_mov_b32_e32 v2, 1.0 +; GFX908-NEXT: v_mov_b32_e32 v3, 2.0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31] +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a24 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v29, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v28, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a0 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:96 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a4 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:112 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:64 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:80 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:32 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[22:25], off offset:48 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[26:29], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: s_setpc_b64 s[30:31] bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0) @@ -105,9 +1030,101 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_nonentry_with_agpr: -; GCN: v_mfma_f32_32x32x1{{.*}} a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9:]+}}, a[{{[0-9:]+}}] define void @test_mfma_f32_32x32x1f32_nonentry_with_agpr(ptr addrspace(1) %arg) #3 { +; GFX908-LABEL: test_mfma_f32_32x32x1f32_nonentry_with_agpr: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX908-NEXT: global_load_dwordx4 v[30:33], v[0:1], off offset:112 +; GFX908-NEXT: global_load_dwordx4 v[26:29], v[0:1], off offset:96 +; GFX908-NEXT: global_load_dwordx4 v[22:25], v[0:1], off offset:80 +; GFX908-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:64 +; GFX908-NEXT: global_load_dwordx4 v[14:17], v[0:1], off offset:48 +; GFX908-NEXT: global_load_dwordx4 v[10:13], v[0:1], off offset:32 +; GFX908-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:16 +; GFX908-NEXT: global_load_dwordx4 v[2:5], v[0:1], off +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a0, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v6 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v7 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v8 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v9 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v10 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v11 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v12 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v13 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v14 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v15 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v16 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v18 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v19 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v20 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v21 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v22 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v23 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v24 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v25 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v26 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v27 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v28 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v29 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v30 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v31 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v32 +; GFX908-NEXT: v_accvgpr_write_b32 a31, v33 +; GFX908-NEXT: v_mov_b32_e32 v2, 1.0 +; GFX908-NEXT: v_mov_b32_e32 v3, 2.0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31] +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a24 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v29, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v28, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a0 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:96 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a4 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:112 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:64 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:80 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:32 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[22:25], off offset:48 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[26:29], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: s_setpc_b64 s[30:31] bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0) @@ -121,3 +1138,6 @@ attributes #0 = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="2 attributes #1 = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="2" } attributes #2 = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-agpr-alloc"="0" } attributes #3 = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="2" } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GCN: {{.*}} +; GFX90A: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll index 9c38d7f..5b0d2d2 100644 --- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll +++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll @@ -3565,13 +3565,13 @@ define amdgpu_kernel void @fneg_v2f32_scalar(ptr addrspace(1) %a, <2 x float> %x ; PACKED-SDAG-LABEL: fneg_v2f32_scalar: ; PACKED-SDAG: ; %bb.0: ; PACKED-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 -; PACKED-SDAG-NEXT: v_mov_b32_e32 v2, 0 +; PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; PACKED-SDAG-NEXT: s_xor_b32 s3, s3, 0x80000000 ; PACKED-SDAG-NEXT: s_xor_b32 s2, s2, 0x80000000 -; PACKED-SDAG-NEXT: v_mov_b32_e32 v0, s2 -; PACKED-SDAG-NEXT: v_mov_b32_e32 v1, s3 -; PACKED-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; PACKED-SDAG-NEXT: v_mov_b32_e32 v2, s2 +; PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s3 +; PACKED-SDAG-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1] ; PACKED-SDAG-NEXT: s_endpgm ; ; PACKED-GISEL-LABEL: fneg_v2f32_scalar: diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll index a5c8f04..f54a383 100644 --- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll +++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -;RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 --stop-after=greedy,1 < %s | FileCheck -check-prefix=REGALLOC-GFX908 %s +;RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 --stop-after=greedy,2 < %s | FileCheck -check-prefix=REGALLOC-GFX908 %s ;RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 --stop-after=prologepilog < %s | FileCheck -check-prefix=PEI-GFX908 %s -;RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a --stop-after=greedy,1 < %s | FileCheck -check-prefix=REGALLOC-GFX90A %s +;RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a --stop-after=greedy,2 < %s | FileCheck -check-prefix=REGALLOC-GFX90A %s ;RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a --stop-after=prologepilog < %s | FileCheck -check-prefix=PEI-GFX90A %s ; Partial reg copy and spill missed during regalloc handled later at frame lowering. @@ -12,17 +12,21 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 { ; REGALLOC-GFX908-NEXT: liveins: $sgpr4_sgpr5 ; REGALLOC-GFX908-NEXT: {{ $}} ; REGALLOC-GFX908-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, undef %6:agpr_32 - ; REGALLOC-GFX908-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:VReg_128 */, def %7 - ; REGALLOC-GFX908-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3538954 /* regdef:VReg_64 */, def %8 - ; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef %15:vreg_64, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) + ; REGALLOC-GFX908-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:VReg_128 */, def %25 + ; REGALLOC-GFX908-NEXT: [[COPY:%[0-9]+]]:av_128 = COPY %25 + ; REGALLOC-GFX908-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3538954 /* regdef:VReg_64 */, def %27 + ; REGALLOC-GFX908-NEXT: SI_SPILL_AV64_SAVE %27, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5) + ; REGALLOC-GFX908-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY [[COPY]] + ; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef %15:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) ; REGALLOC-GFX908-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4) - ; REGALLOC-GFX908-NEXT: [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3 + ; REGALLOC-GFX908-NEXT: [[COPY2:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3 ; REGALLOC-GFX908-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec ; REGALLOC-GFX908-NEXT: [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec - ; REGALLOC-GFX908-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec - ; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX2 undef %17:vreg_64, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1) - ; REGALLOC-GFX908-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]] - ; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef %19:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) + ; REGALLOC-GFX908-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY2]], 0, 0, 0, implicit $mode, implicit $exec + ; REGALLOC-GFX908-NEXT: [[SI_SPILL_AV64_RESTORE:%[0-9]+]]:vreg_64 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5) + ; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX2 undef %17:vreg_64, [[SI_SPILL_AV64_RESTORE]], 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1) + ; REGALLOC-GFX908-NEXT: [[COPY3:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]] + ; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef %19:vreg_64, [[COPY3]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) ; REGALLOC-GFX908-NEXT: S_ENDPGM 0 ; ; PEI-GFX908-LABEL: name: partial_copy @@ -57,40 +61,35 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 { ; REGALLOC-GFX90A-NEXT: liveins: $sgpr4_sgpr5 ; REGALLOC-GFX90A-NEXT: {{ $}} ; REGALLOC-GFX90A-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, undef %6:agpr_32 - ; REGALLOC-GFX90A-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6422538 /* regdef:VReg_128_Align2 */, def %7 - ; REGALLOC-GFX90A-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3866634 /* regdef:VReg_64_Align2 */, def %8 - ; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef %15:vreg_64_align2, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) + ; REGALLOC-GFX90A-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6422538 /* regdef:VReg_128_Align2 */, def %24 + ; REGALLOC-GFX90A-NEXT: [[COPY:%[0-9]+]]:av_128_align2 = COPY %24 + ; REGALLOC-GFX90A-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3866634 /* regdef:VReg_64_Align2 */, def %22 + ; REGALLOC-GFX90A-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY %22 + ; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef %15:vreg_64_align2, [[COPY]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) ; REGALLOC-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4) - ; REGALLOC-GFX90A-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3 + ; REGALLOC-GFX90A-NEXT: [[COPY2:%[0-9]+]]:areg_128_align2 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3 ; REGALLOC-GFX90A-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec ; REGALLOC-GFX90A-NEXT: [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec - ; REGALLOC-GFX90A-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec - ; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX2 undef %17:vreg_64_align2, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1) + ; REGALLOC-GFX90A-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY2]], 0, 0, 0, implicit $mode, implicit $exec + ; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX2 undef %17:vreg_64_align2, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1) ; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef %19:vreg_64_align2, [[V_MFMA_I32_4X4X4I8_e64_]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) ; REGALLOC-GFX90A-NEXT: S_ENDPGM 0 ; ; PEI-GFX90A-LABEL: name: partial_copy ; PEI-GFX90A: bb.0 (%ir-block.0): - ; PEI-GFX90A-NEXT: liveins: $agpr4, $sgpr4_sgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr9 + ; PEI-GFX90A-NEXT: liveins: $sgpr4_sgpr5 ; PEI-GFX90A-NEXT: {{ $}} - ; PEI-GFX90A-NEXT: $sgpr12_sgpr13_sgpr14_sgpr15 = COPY killed $sgpr0_sgpr1_sgpr2_sgpr3 - ; PEI-GFX90A-NEXT: $sgpr12 = S_ADD_U32 $sgpr12, $sgpr9, implicit-def $scc, implicit-def $sgpr12_sgpr13_sgpr14_sgpr15 - ; PEI-GFX90A-NEXT: $sgpr13 = S_ADDC_U32 $sgpr13, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr12_sgpr13_sgpr14_sgpr15 ; PEI-GFX90A-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, undef renamable $agpr0 ; PEI-GFX90A-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6422538 /* regdef:VReg_128_Align2 */, def renamable $vgpr0_vgpr1_vgpr2_vgpr3 ; PEI-GFX90A-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec - ; PEI-GFX90A-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3866634 /* regdef:VReg_64_Align2 */, def renamable $vgpr0_vgpr1 - ; PEI-GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1 :: (store (s32) into %stack.0, addrspace 5) - ; PEI-GFX90A-NEXT: $agpr4 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr1, implicit $exec, implicit killed $vgpr0_vgpr1 + ; PEI-GFX90A-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3866634 /* regdef:VReg_64_Align2 */, def renamable $vgpr2_vgpr3 ; PEI-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $agpr0_agpr1_agpr2_agpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) ; PEI-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4) ; PEI-GFX90A-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec ; PEI-GFX90A-NEXT: renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec ; PEI-GFX90A-NEXT: renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec ; PEI-GFX90A-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec - ; PEI-GFX90A-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5) - ; PEI-GFX90A-NEXT: $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1 - ; PEI-GFX90A-NEXT: GLOBAL_STORE_DWORDX2 undef renamable $vgpr0_vgpr1, killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1) + ; PEI-GFX90A-NEXT: GLOBAL_STORE_DWORDX2 undef renamable $vgpr0_vgpr1, killed renamable $vgpr2_vgpr3, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1) ; PEI-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $agpr0_agpr1_agpr2_agpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) ; PEI-GFX90A-NEXT: S_ENDPGM 0 call void asm sideeffect "; use $0", "a" (i32 poison) diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll index d48bfe0..68ef30a9 100644 --- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll +++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll @@ -53,31 +53,31 @@ define amdgpu_kernel void @store_v16i32(ptr addrspace(1) %out, <16 x i32> %a) { ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40 ; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 -; GFX942-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: v_mov_b32_e32 v0, s20 -; GFX942-NEXT: v_mov_b32_e32 v1, s21 -; GFX942-NEXT: v_mov_b32_e32 v2, s22 -; GFX942-NEXT: v_mov_b32_e32 v3, s23 -; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48 +; GFX942-NEXT: v_mov_b32_e32 v2, s20 +; GFX942-NEXT: v_mov_b32_e32 v3, s21 +; GFX942-NEXT: v_mov_b32_e32 v4, s22 +; GFX942-NEXT: v_mov_b32_e32 v5, s23 +; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:48 ; GFX942-NEXT: s_nop 1 -; GFX942-NEXT: v_mov_b32_e32 v0, s16 -; GFX942-NEXT: v_mov_b32_e32 v1, s17 -; GFX942-NEXT: v_mov_b32_e32 v2, s18 -; GFX942-NEXT: v_mov_b32_e32 v3, s19 -; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32 +; GFX942-NEXT: v_mov_b32_e32 v2, s16 +; GFX942-NEXT: v_mov_b32_e32 v3, s17 +; GFX942-NEXT: v_mov_b32_e32 v4, s18 +; GFX942-NEXT: v_mov_b32_e32 v5, s19 +; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:32 ; GFX942-NEXT: s_nop 1 -; GFX942-NEXT: v_mov_b32_e32 v0, s12 -; GFX942-NEXT: v_mov_b32_e32 v1, s13 -; GFX942-NEXT: v_mov_b32_e32 v2, s14 -; GFX942-NEXT: v_mov_b32_e32 v3, s15 -; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 +; GFX942-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-NEXT: v_mov_b32_e32 v4, s14 +; GFX942-NEXT: v_mov_b32_e32 v5, s15 +; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] offset:16 ; GFX942-NEXT: s_nop 1 -; GFX942-NEXT: v_mov_b32_e32 v0, s8 -; GFX942-NEXT: v_mov_b32_e32 v1, s9 -; GFX942-NEXT: v_mov_b32_e32 v2, s10 -; GFX942-NEXT: v_mov_b32_e32 v3, s11 -; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX942-NEXT: v_mov_b32_e32 v2, s8 +; GFX942-NEXT: v_mov_b32_e32 v3, s9 +; GFX942-NEXT: v_mov_b32_e32 v4, s10 +; GFX942-NEXT: v_mov_b32_e32 v5, s11 +; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] ; GFX942-NEXT: s_endpgm entry: store <16 x i32> %a, ptr addrspace(1) %out diff --git a/llvm/test/CodeGen/AMDGPU/read-write-register-illegal-type.ll b/llvm/test/CodeGen/AMDGPU/read-write-register-illegal-type.ll new file mode 100644 index 0000000..2324f3f --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/read-write-register-illegal-type.ll @@ -0,0 +1,29 @@ +; RUN: not llc -mtriple=amdgcn -mcpu=gfx900 -filetype=null %s 2>&1 | FileCheck %s + +; CHECK: error: <unknown>:0:0: cannot use llvm.read_register with illegal type +define amdgpu_kernel void @test_read_register_i9(ptr addrspace(1) %out) nounwind { + %reg = call i9 @llvm.read_register.i9(metadata !0) + store i9 %reg, ptr addrspace(1) %out + ret void +} + +; CHECK: error: <unknown>:0:0: cannot use llvm.write_register with illegal type +define amdgpu_kernel void @test_write_register_i9(ptr addrspace(1) %out) nounwind { + call void @llvm.write_register.i9(metadata !0, i9 42) + ret void +} + +; CHECK: error: <unknown>:0:0: cannot use llvm.read_register with illegal type +define amdgpu_kernel void @test_read_register_i128(ptr addrspace(1) %out) nounwind { + %reg = call i128 @llvm.read_register.i128(metadata !0) + store i128 %reg, ptr addrspace(1) %out + ret void +} + +; CHECK: error: <unknown>:0:0: cannot use llvm.write_register with illegal type +define amdgpu_kernel void @test_write_register_i128(ptr addrspace(1) %out) nounwind { + call void @llvm.write_register.i128(metadata !0, i128 42) + ret void +} + +!0 = !{!"m0"} diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-illegal-eviction-assert.ll b/llvm/test/CodeGen/AMDGPU/regalloc-illegal-eviction-assert.ll index f2fd3a8..c035e9f 100644 --- a/llvm/test/CodeGen/AMDGPU/regalloc-illegal-eviction-assert.ll +++ b/llvm/test/CodeGen/AMDGPU/regalloc-illegal-eviction-assert.ll @@ -9,9 +9,9 @@ %asm.output = type { <16 x i32>, <8 x i32>, <5 x i32>, <4 x i32>, <16 x i32> } ; CHECK-LABEL: {{^}}illegal_eviction_assert: -; CHECK: ; def v[4:19] v[20:27] v[0:4] v[0:3] a[0:15] +; CHECK: ; def v[13:28] v[0:7] v[8:12] v[0:3] a[0:15] ; CHECK: ; clobber -; CHECK: ; use v[4:19] v[20:27] v[0:4] v[0:3] a[1:16] +; CHECK: ; use v[13:28] v[0:7] v[8:12] v[0:3] a[1:16] define void @illegal_eviction_assert(ptr addrspace(1) %arg) #0 { ;%agpr0 = call i32 asm sideeffect "; def $0","=${a0}"() %asm = call %asm.output asm sideeffect "; def $0 $1 $2 $3 $4","=v,=v,=v,=v,={a[0:15]}"() diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll index e8e122e..bbb9df9 100644 --- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll +++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll @@ -59,19 +59,19 @@ define amdgpu_kernel void @scalar_to_vector_v8i16(<2 x i32> %in, ptr %out) #0 { ; GFX90A-LABEL: scalar_to_vector_v8i16: ; GFX90A: ; %bb.0: ; %entry ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 -; GFX90A-NEXT: v_and_b32_e32 v4, 0x3ff, v0 -; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 4, v4 +; GFX90A-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; GFX90A-NEXT: s_add_u32 flat_scratch_lo, s12, s17 ; GFX90A-NEXT: s_addc_u32 flat_scratch_hi, s13, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v5, s3 -; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, s2, v4 -; GFX90A-NEXT: v_mov_b32_e32 v0, s0 -; GFX90A-NEXT: v_mov_b32_e32 v1, s1 +; GFX90A-NEXT: v_mov_b32_e32 v1, s3 +; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0 ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 -; GFX90A-NEXT: v_mov_b32_e32 v3, s0 -; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc -; GFX90A-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; GFX90A-NEXT: v_mov_b32_e32 v3, s1 +; GFX90A-NEXT: v_mov_b32_e32 v4, s0 +; GFX90A-NEXT: v_mov_b32_e32 v5, s0 +; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX90A-NEXT: flat_store_dwordx4 v[0:1], v[2:5] ; GFX90A-NEXT: s_endpgm entry: %val.1.i32 = extractelement <2 x i32> %in, i64 0 @@ -146,19 +146,19 @@ define amdgpu_kernel void @scalar_to_vector_v8f16(<2 x float> %in, ptr %out) #0 ; GFX90A-LABEL: scalar_to_vector_v8f16: ; GFX90A: ; %bb.0: ; %entry ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 -; GFX90A-NEXT: v_and_b32_e32 v4, 0x3ff, v0 -; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 4, v4 +; GFX90A-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 4, v0 ; GFX90A-NEXT: s_add_u32 flat_scratch_lo, s12, s17 ; GFX90A-NEXT: s_addc_u32 flat_scratch_hi, s13, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v5, s3 -; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, s2, v4 -; GFX90A-NEXT: v_mov_b32_e32 v0, s0 -; GFX90A-NEXT: v_mov_b32_e32 v1, s1 -; GFX90A-NEXT: v_mov_b32_e32 v3, s0 +; GFX90A-NEXT: v_mov_b32_e32 v1, s3 +; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0 ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 -; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc -; GFX90A-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; GFX90A-NEXT: v_mov_b32_e32 v3, s1 +; GFX90A-NEXT: v_mov_b32_e32 v5, s0 +; GFX90A-NEXT: v_mov_b32_e32 v4, s0 +; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX90A-NEXT: flat_store_dwordx4 v[0:1], v[2:5] ; GFX90A-NEXT: s_endpgm entry: %val.1.float = extractelement <2 x float> %in, i64 0 diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll b/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll index 9361187..4d864ad 100644 --- a/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll +++ b/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll @@ -25,27 +25,27 @@ define void @shufflevector_v2i32_10_physreg_even_vgpr_pair_copy(ptr addrspace(1) ; GFX90A-LABEL: shufflevector_v2i32_10_physreg_even_vgpr_pair_copy: ; GFX90A: ; %bb.0: ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v4, v5 ; GFX90A-NEXT: ;;#ASMEND -; GFX90A-NEXT: v_mov_b32_e32 v0, v5 -; GFX90A-NEXT: v_mov_b32_e32 v1, v4 -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[16:17] +; GFX90A-NEXT: v_mov_b32_e32 v2, v5 +; GFX90A-NEXT: v_mov_b32_e32 v3, v4 +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[16:17] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: shufflevector_v2i32_10_physreg_even_vgpr_pair_copy: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-NEXT: v_mov_b32_e32 v0, 0 ; GFX940-NEXT: ;;#ASMSTART ; GFX940-NEXT: ; def v4, v5 ; GFX940-NEXT: ;;#ASMEND ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_mov_b32_e32 v0, v5 -; GFX940-NEXT: v_mov_b32_e32 v1, v4 -; GFX940-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX940-NEXT: v_mov_b32_e32 v2, v5 +; GFX940-NEXT: v_mov_b32_e32 v3, v4 +; GFX940-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: s_setpc_b64 s[30:31] %asm = call { i32, i32 } asm "; def $0, $1", "={v4},={v5}"() @@ -214,27 +214,27 @@ define void @shufflevector_v2i32_11_physreg_even_vgpr_pair_copy(ptr addrspace(1) ; GFX90A-LABEL: shufflevector_v2i32_11_physreg_even_vgpr_pair_copy: ; GFX90A: ; %bb.0: ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v4, v5 ; GFX90A-NEXT: ;;#ASMEND -; GFX90A-NEXT: v_mov_b32_e32 v0, v5 -; GFX90A-NEXT: v_mov_b32_e32 v1, v5 -; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[16:17] +; GFX90A-NEXT: v_mov_b32_e32 v2, v5 +; GFX90A-NEXT: v_mov_b32_e32 v3, v5 +; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[16:17] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: shufflevector_v2i32_11_physreg_even_vgpr_pair_copy: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-NEXT: v_mov_b32_e32 v0, 0 ; GFX940-NEXT: ;;#ASMSTART ; GFX940-NEXT: ; def v4, v5 ; GFX940-NEXT: ;;#ASMEND ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_mov_b32_e32 v0, v5 -; GFX940-NEXT: v_mov_b32_e32 v1, v5 -; GFX940-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX940-NEXT: v_mov_b32_e32 v2, v5 +; GFX940-NEXT: v_mov_b32_e32 v3, v5 +; GFX940-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: s_setpc_b64 s[30:31] %asm = call { i32, i32 } asm "; def $0, $1", "={v4},={v5}"() @@ -265,31 +265,31 @@ define void @shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy(ptr addrspace( ; GFX90A-LABEL: shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy: ; GFX90A: ; %bb.0: ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v4, v5, v6, v7 ; GFX90A-NEXT: ;;#ASMEND -; GFX90A-NEXT: v_mov_b32_e32 v2, v5 -; GFX90A-NEXT: v_mov_b32_e32 v1, v6 -; GFX90A-NEXT: v_mov_b32_e32 v0, v7 -; GFX90A-NEXT: v_mov_b32_e32 v3, v4 -; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17] +; GFX90A-NEXT: v_mov_b32_e32 v10, v5 +; GFX90A-NEXT: v_mov_b32_e32 v9, v6 +; GFX90A-NEXT: v_mov_b32_e32 v8, v7 +; GFX90A-NEXT: v_mov_b32_e32 v11, v4 +; GFX90A-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_mov_b32_e32 v8, 0 +; GFX940-NEXT: v_mov_b32_e32 v0, 0 ; GFX940-NEXT: ;;#ASMSTART ; GFX940-NEXT: ; def v4, v5, v6, v7 ; GFX940-NEXT: ;;#ASMEND ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_mov_b32_e32 v2, v5 -; GFX940-NEXT: v_mov_b32_e32 v1, v6 -; GFX940-NEXT: v_mov_b32_e32 v0, v7 -; GFX940-NEXT: v_mov_b32_e32 v3, v4 -; GFX940-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX940-NEXT: v_mov_b32_e32 v10, v5 +; GFX940-NEXT: v_mov_b32_e32 v9, v6 +; GFX940-NEXT: v_mov_b32_e32 v8, v7 +; GFX940-NEXT: v_mov_b32_e32 v11, v4 +; GFX940-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: s_setpc_b64 s[30:31] %asm = call { i32, i32, i32, i32 } asm "; def $0, $1, $2, $3", "={v4},={v5},={v6},={v7}"() @@ -327,31 +327,31 @@ define void @shufflevector_v4i32_1032_physreg_even_vgpr_quad_copy(ptr addrspace( ; GFX90A-LABEL: shufflevector_v4i32_1032_physreg_even_vgpr_quad_copy: ; GFX90A: ; %bb.0: ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v4, v5, v6, v7 ; GFX90A-NEXT: ;;#ASMEND -; GFX90A-NEXT: v_mov_b32_e32 v0, v5 -; GFX90A-NEXT: v_mov_b32_e32 v3, v6 -; GFX90A-NEXT: v_mov_b32_e32 v2, v7 -; GFX90A-NEXT: v_mov_b32_e32 v1, v4 -; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17] +; GFX90A-NEXT: v_mov_b32_e32 v8, v5 +; GFX90A-NEXT: v_mov_b32_e32 v11, v6 +; GFX90A-NEXT: v_mov_b32_e32 v10, v7 +; GFX90A-NEXT: v_mov_b32_e32 v9, v4 +; GFX90A-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: shufflevector_v4i32_1032_physreg_even_vgpr_quad_copy: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_mov_b32_e32 v8, 0 +; GFX940-NEXT: v_mov_b32_e32 v0, 0 ; GFX940-NEXT: ;;#ASMSTART ; GFX940-NEXT: ; def v4, v5, v6, v7 ; GFX940-NEXT: ;;#ASMEND ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_mov_b32_e32 v0, v5 -; GFX940-NEXT: v_mov_b32_e32 v3, v6 -; GFX940-NEXT: v_mov_b32_e32 v2, v7 -; GFX940-NEXT: v_mov_b32_e32 v1, v4 -; GFX940-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX940-NEXT: v_mov_b32_e32 v8, v5 +; GFX940-NEXT: v_mov_b32_e32 v11, v6 +; GFX940-NEXT: v_mov_b32_e32 v10, v7 +; GFX940-NEXT: v_mov_b32_e32 v9, v4 +; GFX940-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: s_setpc_b64 s[30:31] %asm = call { i32, i32, i32, i32 } asm "; def $0, $1, $2, $3", "={v4},={v5},={v6},={v7}"() @@ -746,16 +746,15 @@ define i32 @shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy_other_use_elt(p ; GFX90A-LABEL: shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy_other_use_elt: ; GFX90A: ; %bb.0: ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v4, v5, v6, v7 ; GFX90A-NEXT: ;;#ASMEND -; GFX90A-NEXT: v_mov_b32_e32 v2, v5 -; GFX90A-NEXT: v_mov_b32_e32 v1, v6 -; GFX90A-NEXT: v_mov_b32_e32 v0, v7 -; GFX90A-NEXT: v_mov_b32_e32 v3, v4 -; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[16:17] -; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: v_mov_b32_e32 v10, v5 +; GFX90A-NEXT: v_mov_b32_e32 v9, v6 +; GFX90A-NEXT: v_mov_b32_e32 v8, v7 +; GFX90A-NEXT: v_mov_b32_e32 v11, v4 +; GFX90A-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17] ; GFX90A-NEXT: v_mov_b32_e32 v0, v6 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] @@ -763,17 +762,16 @@ define i32 @shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy_other_use_elt(p ; GFX940-LABEL: shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy_other_use_elt: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_mov_b32_e32 v8, 0 +; GFX940-NEXT: v_mov_b32_e32 v0, 0 ; GFX940-NEXT: ;;#ASMSTART ; GFX940-NEXT: ; def v4, v5, v6, v7 ; GFX940-NEXT: ;;#ASMEND ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_mov_b32_e32 v2, v5 -; GFX940-NEXT: v_mov_b32_e32 v1, v6 -; GFX940-NEXT: v_mov_b32_e32 v0, v7 -; GFX940-NEXT: v_mov_b32_e32 v3, v4 -; GFX940-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] -; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_mov_b32_e32 v10, v5 +; GFX940-NEXT: v_mov_b32_e32 v9, v6 +; GFX940-NEXT: v_mov_b32_e32 v8, v7 +; GFX940-NEXT: v_mov_b32_e32 v11, v4 +; GFX940-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1] ; GFX940-NEXT: v_mov_b32_e32 v0, v6 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: s_setpc_b64 s[30:31] diff --git a/llvm/test/CodeGen/AMDGPU/spill-agpr.ll b/llvm/test/CodeGen/AMDGPU/spill-agpr.ll index 5484f77..eb0d546 100644 --- a/llvm/test/CodeGen/AMDGPU/spill-agpr.ll +++ b/llvm/test/CodeGen/AMDGPU/spill-agpr.ll @@ -1,15 +1,107 @@ -; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX908 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX90A %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -check-prefixes=GCN,GFX908 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -check-prefixes=GCN,GFX90A %s -; GCN-LABEL: {{^}}max_12regs_13a_used: -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1 -; GCN: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a{{[0-9]+}} -; GCN-NOT: buffer_store_dword -; GCN-NOT: buffer_load_dword -; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]] -; GCN: ScratchSize: 0 define amdgpu_kernel void @max_12regs_13a_used(i32 %cond, ptr addrspace(1) %arg, ptr addrspace(1) %out) #2 { +; GFX908-LABEL: max_12regs_13a_used: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX908-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-NEXT: s_load_dword s0, s[0:1], 0x24 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_cmp_lg_u32 s0, 0 +; GFX908-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v5, s8 +; GFX908-NEXT: v_mov_b32_e32 v1, s9 +; GFX908-NEXT: v_mov_b32_e32 v2, s10 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-NEXT: v_mov_b32_e32 v5, s11 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v0, a[0:3] +; GFX908-NEXT: v_mfma_f32_4x4x1f32 a[4:7], v0, v0, a[0:3] +; GFX908-NEXT: s_cbranch_scc0 .LBB0_2 +; GFX908-NEXT: ; %bb.1: ; %st +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: s_endpgm +; GFX908-NEXT: .LBB0_2: ; %use +; GFX908-NEXT: s_nop 2 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a4 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a6 +; GFX908-NEXT: v_accvgpr_write_b32 a4, 4 +; GFX908-NEXT: v_accvgpr_write_b32 a8, 5 +; GFX908-NEXT: v_accvgpr_write_b32 a9, 1 +; GFX908-NEXT: v_accvgpr_write_b32 a10, 2 +; GFX908-NEXT: v_accvgpr_write_b32 a11, 3 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: v_accvgpr_write_b32 a7, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-NEXT: v_mov_b32_e32 v1, v0 +; GFX908-NEXT: v_mov_b32_e32 v2, v0 +; GFX908-NEXT: v_mov_b32_e32 v3, v0 +; GFX908-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: s_endpgm +; +; GFX90A-LABEL: max_12regs_13a_used: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX90A-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_cmp_lg_u32 s0, 0 +; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v0, a[0:3] +; GFX90A-NEXT: v_mfma_f32_4x4x1f32 a[4:7], v0, v0, a[0:3] +; GFX90A-NEXT: s_cbranch_scc0 .LBB0_2 +; GFX90A-NEXT: ; %bb.1: ; %st +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: s_endpgm +; GFX90A-NEXT: .LBB0_2: ; %use +; GFX90A-NEXT: s_nop 3 +; GFX90A-NEXT: v_accvgpr_read_b32 v9, a7 +; GFX90A-NEXT: v_accvgpr_read_b32 v8, a6 +; GFX90A-NEXT: v_accvgpr_read_b32 v7, a5 +; GFX90A-NEXT: v_accvgpr_read_b32 v6, a4 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, 4 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, 5 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, 1 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, 2 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, 3 +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: v_accvgpr_write_b32 a4, v6 +; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, v7 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, v8 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, v9 +; GFX90A-NEXT: v_mov_b32_e32 v1, v0 +; GFX90A-NEXT: v_mov_b32_e32 v2, v0 +; GFX90A-NEXT: v_mov_b32_e32 v3, v0 +; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX90A-NEXT: s_waitcnt vmcnt(0) +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 1.0, <4 x float> %in.1, i32 0, i32 0, i32 0) @@ -28,16 +120,64 @@ st: call void asm sideeffect "", "a,a"(<4 x float> %mai.1, <4 x float> %mai.2) ret void } - -; GCN-LABEL: {{^}}max_10_vgprs_used_9a: -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1 -; GCN: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a{{[0-9]+}} -; GCN-NOT: buffer_store_dword -; GCN-NOT: buffer_load_dword -; GCN: v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]] ; GCN: ScratchSize: 0 + define amdgpu_kernel void @max_10_vgprs_used_9a() #1 { +; GFX908-LABEL: max_10_vgprs_used_9a: +; GFX908: ; %bb.0: +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: v_accvgpr_read_b32 v5, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a1 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v1 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: v_accvgpr_write_b32 a0, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v5 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: s_endpgm +; +; GFX90A-LABEL: max_10_vgprs_used_9a: +; GFX90A: ; %bb.0: +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX90A-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX90A-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: v_accvgpr_read_b32 v5, a3 +; GFX90A-NEXT: v_accvgpr_read_b32 v4, a2 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, v3 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, v2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, v1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, v0 +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: v_accvgpr_write_b32 a0, v4 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, v5 +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: s_endpgm %a1 = call <4 x i32> asm sideeffect "", "=a"() %a2 = call <4 x i32> asm sideeffect "", "=a"() %a3 = call i32 asm sideeffect "", "=a"() @@ -46,17 +186,168 @@ define amdgpu_kernel void @max_10_vgprs_used_9a() #1 { call void asm sideeffect "", "a"(<2 x i32> %a4) ret void } - -; GCN-LABEL: {{^}}max_32regs_mfma32: -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1 -; GCN-NOT: buffer_store_dword -; GCN: v_accvgpr_read_b32 -; GCN: v_mfma_f32_32x32x1f32 -; GCN-NOT: buffer_load_dword -; GCN: v_accvgpr_write_b32 ; GCN: ScratchSize: 0 + define amdgpu_kernel void @max_32regs_mfma32(ptr addrspace(1) %arg) #3 { +; GFX908-LABEL: max_32regs_mfma32: +; GFX908: ; %bb.0: ; %bb +; GFX908-NEXT: v_mov_b32_e32 v2, 0x40400000 +; GFX908-NEXT: v_mov_b32_e32 v3, 0x40c00000 +; GFX908-NEXT: v_mov_b32_e32 v4, 0x40e00000 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x40a00000 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v2 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41000000 +; GFX908-NEXT: v_mov_b32_e32 v3, 0x41100000 +; GFX908-NEXT: v_mov_b32_e32 v4, 0x41200000 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v4 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41300000 +; GFX908-NEXT: v_mov_b32_e32 v3, 0x41400000 +; GFX908-NEXT: v_mov_b32_e32 v4, 0x41500000 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v4 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41600000 +; GFX908-NEXT: v_mov_b32_e32 v3, 0x41700000 +; GFX908-NEXT: v_mov_b32_e32 v4, 0x41800000 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v4 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41880000 +; GFX908-NEXT: v_mov_b32_e32 v3, 0x41900000 +; GFX908-NEXT: v_mov_b32_e32 v4, 0x41980000 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v4 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41a00000 +; GFX908-NEXT: v_mov_b32_e32 v3, 0x41a80000 +; GFX908-NEXT: v_mov_b32_e32 v4, 0x41b00000 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v4 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41b80000 +; GFX908-NEXT: v_mov_b32_e32 v3, 0x41c00000 +; GFX908-NEXT: v_mov_b32_e32 v4, 0x41c80000 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v4 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41d00000 +; GFX908-NEXT: v_mov_b32_e32 v3, 0x41d80000 +; GFX908-NEXT: v_mov_b32_e32 v4, 0x41e00000 +; GFX908-NEXT: v_mov_b32_e32 v1, 1.0 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v4 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41e80000 +; GFX908-NEXT: v_mov_b32_e32 v3, 0x41f00000 +; GFX908-NEXT: v_mov_b32_e32 v4, 0x41f80000 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: v_accvgpr_read_b32 v5, a0 +; GFX908-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX908-NEXT: v_accvgpr_write_b32 a1, 2.0 +; GFX908-NEXT: v_accvgpr_write_b32 a3, 4.0 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a31, 2.0 +; GFX908-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v1, a[0:31] +; GFX908-NEXT: v_mov_b32_e32 v0, 0 +; GFX908-NEXT: s_nop 7 +; GFX908-NEXT: s_nop 5 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v5 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: global_store_dword v0, v1, s[2:3] +; GFX908-NEXT: s_endpgm +; +; GFX90A-LABEL: max_32regs_mfma32: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x40400000 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x40a00000 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x40c00000 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x40e00000 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41000000 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41100000 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41200000 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41300000 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41400000 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41500000 +; GFX90A-NEXT: v_accvgpr_write_b32 a12, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41600000 +; GFX90A-NEXT: v_accvgpr_write_b32 a13, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41700000 +; GFX90A-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41800000 +; GFX90A-NEXT: v_accvgpr_write_b32 a15, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41880000 +; GFX90A-NEXT: v_accvgpr_write_b32 a16, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41900000 +; GFX90A-NEXT: v_accvgpr_write_b32 a17, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41980000 +; GFX90A-NEXT: v_accvgpr_write_b32 a18, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41a00000 +; GFX90A-NEXT: v_accvgpr_write_b32 a19, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41a80000 +; GFX90A-NEXT: v_accvgpr_write_b32 a20, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41b00000 +; GFX90A-NEXT: v_accvgpr_write_b32 a21, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41b80000 +; GFX90A-NEXT: v_accvgpr_write_b32 a22, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41c00000 +; GFX90A-NEXT: v_accvgpr_write_b32 a23, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41c80000 +; GFX90A-NEXT: v_accvgpr_write_b32 a24, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41d00000 +; GFX90A-NEXT: v_accvgpr_write_b32 a25, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41d80000 +; GFX90A-NEXT: v_mov_b32_e32 v1, 1.0 +; GFX90A-NEXT: v_accvgpr_write_b32 a26, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41e00000 +; GFX90A-NEXT: v_accvgpr_write_b32 a27, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41e80000 +; GFX90A-NEXT: v_accvgpr_write_b32 a28, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41f00000 +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: v_accvgpr_write_b32 a1, 2.0 +; GFX90A-NEXT: v_accvgpr_write_b32 a29, v2 +; GFX90A-NEXT: v_mov_b32_e32 v2, 0x41f80000 +; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, 4.0 +; GFX90A-NEXT: v_accvgpr_write_b32 a30, v2 +; GFX90A-NEXT: v_accvgpr_mov_b32 a31, a1 +; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v1, a[0:31] +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3 +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: global_store_dword v0, a0, s[2:3] +; GFX90A-NEXT: s_endpgm bb: %v = call i32 asm sideeffect "", "=a"() br label %use @@ -68,42 +359,110 @@ use: store float %elt1, ptr addrspace(1) %arg ret void } +; GCN: ScratchSize: 0 ; Should spill agprs to memory for both gfx908 and gfx90a. -; GCN-LABEL: {{^}}max_6regs_used_8a: -; GCN: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0 -; GCN: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1 - -; GFX908-DAG: v_accvgpr_read_b32 v5, a0 ; Reload Reuse -; GFX908-DAG: buffer_store_dword v5, off, s[{{[0-9:]+}}], 0 ; 4-byte Folded Spill -; GFX908-DAG: v_accvgpr_read_b32 v5, a1 ; Reload Reuse -; GFX908-DAG: buffer_store_dword v5, off, s[{{[0-9:]+}}], 0 offset:4 ; 4-byte Folded Spill -; GFX908-DAG: v_accvgpr_read_b32 v5, a2 ; Reload Reuse -; GFX908-DAG: buffer_store_dword v5, off, s[{{[0-9:]+}}], 0 offset:8 ; 4-byte Folded Spill -; GFX908-DAG: v_accvgpr_read_b32 v5, a3 ; Reload Reuse -; GFX908-DAG: buffer_store_dword v5, off, s[{{[0-9:]+}}], 0 offset:12 ; 4-byte Folded Spill - -; GFX90A-DAG: buffer_store_dword a0, off, s[{{[0-9:]+}}], 0 ; 4-byte Folded Spill -; GFX90A-DAG: buffer_store_dword a1, off, s[{{[0-9:]+}}], 0 offset:4 ; 4-byte Folded Spill -; GFX90A-DAG: buffer_store_dword a2, off, s[{{[0-9:]+}}], 0 offset:8 ; 4-byte Folded Spill -; GFX90A-DAG: buffer_store_dword a3, off, s[{{[0-9:]+}}], 0 offset:12 ; 4-byte Folded Spill - -; GCN: v_mfma_f32_4x4x1f32 a[0:3], v{{[0-9]+}}, v{{[0-9]+}}, a[0:3] - -; GFX908-DAG: buffer_load_dword v0, off, s[{{[0-9:]+}}], 0 ; 4-byte Folded Reload -; GFX908-DAG: buffer_load_dword v1, off, s[{{[0-9:]+}}], 0 offset:4 ; 4-byte Folded Reload -; GFX908-DAG: buffer_load_dword v2, off, s[{{[0-9:]+}}], 0 offset:8 ; 4-byte Folded Reload -; GFX908-DAG: buffer_load_dword v3, off, s[{{[0-9:]+}}], 0 offset:12 ; 4-byte Folded Reload -; GFX908: global_store_dwordx4 v[{{[0-9:]+}}], v[0:3], off - -; GFX90A-DAG: buffer_load_dword v2, off, s[4:7], 0 ; 4-byte Folded Reload -; GFX90A-DAG: buffer_load_dword v3, off, s[4:7], 0 offset:4 ; 4-byte Folded Reload -; GFX90A-DAG: buffer_load_dword v4, off, s[4:7], 0 offset:8 ; 4-byte Folded Reload -; GFX90A-DAG: buffer_load_dword v5, off, s[4:7], 0 offset:12 ; 4-byte Folded Reload -; GFX90A: global_store_dwordx4 v[0:1], v[2:5], off - -; GCN: ScratchSize: 20 define amdgpu_kernel void @max_6regs_used_8a(ptr addrspace(1) %arg) #4 { +; GFX908-LABEL: max_6regs_used_8a: +; GFX908: ; %bb.0: +; GFX908-NEXT: s_mov_b32 s4, SCRATCH_RSRC_DWORD0 +; GFX908-NEXT: s_mov_b32 s5, SCRATCH_RSRC_DWORD1 +; GFX908-NEXT: s_mov_b32 s6, -1 +; GFX908-NEXT: s_mov_b32 s7, 0xe00000 +; GFX908-NEXT: s_add_u32 s4, s4, s3 +; GFX908-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ; def v1 +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: v_lshlrev_b32_e32 v4, 4, v0 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ; def a[0:3] +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: s_addc_u32 s5, s5, 0 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: global_load_dwordx4 v[0:3], v4, s[2:3] +; GFX908-NEXT: v_accvgpr_read_b32 v5, a0 ; Reload Reuse +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: buffer_store_dword v5, off, s[4:7], 0 ; 4-byte Folded Spill +; GFX908-NEXT: v_accvgpr_read_b32 v5, a1 ; Reload Reuse +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: buffer_store_dword v5, off, s[4:7], 0 offset:4 ; 4-byte Folded Spill +; GFX908-NEXT: v_accvgpr_read_b32 v5, a2 ; Reload Reuse +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: buffer_store_dword v5, off, s[4:7], 0 offset:8 ; 4-byte Folded Spill +; GFX908-NEXT: v_accvgpr_read_b32 v5, a3 ; Reload Reuse +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: buffer_store_dword v5, off, s[4:7], 0 offset:12 ; 4-byte Folded Spill +; GFX908-NEXT: s_waitcnt vmcnt(4) +; GFX908-NEXT: v_accvgpr_write_b32 a0, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a3, v3 +; GFX908-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v0, a[0:3] +; GFX908-NEXT: s_nop 3 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: s_nop 1 +; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] +; GFX908-NEXT: buffer_load_dword v0, off, s[4:7], 0 ; 4-byte Folded Reload +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: buffer_load_dword v1, off, s[4:7], 0 offset:4 ; 4-byte Folded Reload +; GFX908-NEXT: buffer_load_dword v2, off, s[4:7], 0 offset:8 ; 4-byte Folded Reload +; GFX908-NEXT: buffer_load_dword v3, off, s[4:7], 0 offset:12 ; 4-byte Folded Reload +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 +; GFX908-NEXT: ;;#ASMSTART +; GFX908-NEXT: ; use v0 +; GFX908-NEXT: ;;#ASMEND +; GFX908-NEXT: s_endpgm +; +; GFX90A-LABEL: max_6regs_used_8a: +; GFX90A: ; %bb.0: +; GFX90A-NEXT: s_mov_b32 s4, SCRATCH_RSRC_DWORD0 +; GFX90A-NEXT: s_mov_b32 s5, SCRATCH_RSRC_DWORD1 +; GFX90A-NEXT: s_mov_b32 s6, -1 +; GFX90A-NEXT: s_mov_b32 s7, 0xe00000 +; GFX90A-NEXT: s_add_u32 s4, s4, s3 +; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX90A-NEXT: s_addc_u32 s5, s5, 0 +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ; def v1 +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ; def a[0:3] +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: buffer_store_dword a0, off, s[4:7], 0 ; 4-byte Folded Spill +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: buffer_store_dword a1, off, s[4:7], 0 offset:4 ; 4-byte Folded Spill +; GFX90A-NEXT: buffer_store_dword a2, off, s[4:7], 0 offset:8 ; 4-byte Folded Spill +; GFX90A-NEXT: buffer_store_dword a3, off, s[4:7], 0 offset:12 ; 4-byte Folded Spill +; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 4, v0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: global_load_dwordx4 a[0:3], v0, s[2:3] +; GFX90A-NEXT: v_mov_b32_e32 v2, 1.0 +; GFX90A-NEXT: s_waitcnt vmcnt(0) +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v2, v2, a[0:3] +; GFX90A-NEXT: s_nop 4 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[2:3] +; GFX90A-NEXT: buffer_load_dword v2, off, s[4:7], 0 ; 4-byte Folded Reload +; GFX90A-NEXT: buffer_load_dword v3, off, s[4:7], 0 offset:4 ; 4-byte Folded Reload +; GFX90A-NEXT: buffer_load_dword v4, off, s[4:7], 0 offset:8 ; 4-byte Folded Reload +; GFX90A-NEXT: buffer_load_dword v5, off, s[4:7], 0 offset:12 ; 4-byte Folded Reload +; GFX90A-NEXT: s_waitcnt vmcnt(0) +; GFX90A-NEXT: global_store_dwordx4 v[0:1], v[2:5], off +; GFX90A-NEXT: s_waitcnt vmcnt(0) +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ; use v1 +; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: s_endpgm %tid = call i32 @llvm.amdgcn.workitem.id.x() %v0 = call float asm sideeffect "; def $0", "=v"() %a4 = call <4 x float> asm sideeffect "; def $0", "=a"() @@ -115,6 +474,7 @@ define amdgpu_kernel void @max_6regs_used_8a(ptr addrspace(1) %arg) #4 { call void asm sideeffect "; use $0", "v"(float %v0); ret void } +; GCN: ScratchSize: 20 declare i32 @llvm.amdgcn.workitem.id.x() declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float, float, <16 x float>, i32, i32, i32) @@ -125,3 +485,5 @@ attributes #1 = { nounwind "amdgpu-num-vgpr"="10" "amdgpu-no-dispatch-id" "amdgp attributes #2 = { nounwind "amdgpu-num-vgpr"="12" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" } attributes #3 = { nounwind "amdgpu-num-vgpr"="32" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" } attributes #4 = { nounwind "amdgpu-num-vgpr"="6" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GCN: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll b/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll new file mode 100644 index 0000000..42436a1 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll @@ -0,0 +1,180 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX900 %s + + +%pair = type { i32, i32 } + +define void @test_extractvalue_then_else(ptr %ptr, i1 %cond) { +; GFX900-LABEL: test_extractvalue_then_else: +; GFX900: ; %bb.0: ; %if +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: flat_load_dword v3, v[0:1] +; GFX900-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX900-NEXT: v_cmp_ne_u32_e32 vcc, 1, v2 +; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX900-NEXT: s_xor_b64 s[4:5], exec, s[4:5] +; GFX900-NEXT: s_cbranch_execz .LBB0_2 +; GFX900-NEXT: ; %bb.1: ; %else +; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_add_u32_e32 v3, 1, v3 +; GFX900-NEXT: .LBB0_2: ; %Flow +; GFX900-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] +; GFX900-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX900-NEXT: flat_store_dword v[0:1], v3 +; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_setpc_b64 s[30:31] +if: + %load_then = load %pair, ptr %ptr + br i1 %cond, label %then, label %else + +then: + %a_then = extractvalue %pair %load_then, 0 + br label %merge + +else: + %a_else = extractvalue %pair %load_then, 0 + %sum_else = add i32 %a_else, 1 + br label %merge + +merge: + %phi = phi i32 [ %a_then, %then ], [ %sum_else, %else ] + store i32 %phi, ptr %ptr + ret void +} + +define void @test_extractvalue_else_then(ptr %ptr, i1 %cond) { +; GFX900-LABEL: test_extractvalue_else_then: +; GFX900: ; %bb.0: ; %if +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: flat_load_dword v3, v[0:1] +; GFX900-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX900-NEXT: v_cmp_ne_u32_e32 vcc, 1, v2 +; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX900-NEXT: s_xor_b64 s[4:5], exec, s[4:5] +; GFX900-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] +; GFX900-NEXT: s_cbranch_execz .LBB1_2 +; GFX900-NEXT: ; %bb.1: ; %else +; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_add_u32_e32 v3, 1, v3 +; GFX900-NEXT: .LBB1_2: ; %merge +; GFX900-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX900-NEXT: flat_store_dword v[0:1], v3 +; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_setpc_b64 s[30:31] +if: + %load_then = load %pair, ptr %ptr + br i1 %cond, label %else, label %then + +else: + %a_else = extractvalue %pair %load_then, 0 + %sum_else = add i32 %a_else, 1 + br label %merge + +then: + %a_then = extractvalue %pair %load_then, 0 + br label %merge + +merge: + %phi = phi i32 [ %a_then, %then ], [ %sum_else, %else ] + store i32 %phi, ptr %ptr + ret void +} + +define amdgpu_kernel void @test_loop_with_if( ptr %ptr, i1 %cond) #0 { +; GFX900-LABEL: test_loop_with_if: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_load_dword s2, s[4:5], 0x2c +; GFX900-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX900-NEXT: v_mov_b32_e32 v5, 0 +; GFX900-NEXT: s_mov_b64 s[4:5], 0 +; GFX900-NEXT: s_movk_i32 s10, 0xfe +; GFX900-NEXT: s_waitcnt lgkmcnt(0) +; GFX900-NEXT: s_bitcmp1_b32 s2, 0 +; GFX900-NEXT: s_cselect_b64 s[2:3], -1, 0 +; GFX900-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[2:3] +; GFX900-NEXT: v_mov_b32_e32 v2, s1 +; GFX900-NEXT: s_xor_b64 s[2:3], s[2:3], -1 +; GFX900-NEXT: v_mov_b32_e32 v1, s0 +; GFX900-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v3 +; GFX900-NEXT: s_branch .LBB2_2 +; GFX900-NEXT: .LBB2_1: ; %latch +; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1 +; GFX900-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_add_u32_e32 v5, 20, v3 +; GFX900-NEXT: v_cmp_lt_i32_e32 vcc, s10, v5 +; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX900-NEXT: flat_store_dword v[1:2], v3 +; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX900-NEXT: s_cbranch_execz .LBB2_8 +; GFX900-NEXT: .LBB2_2: ; %loop +; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX900-NEXT: flat_load_dwordx2 v[3:4], v[1:2] +; GFX900-NEXT: s_and_b64 vcc, exec, s[0:1] +; GFX900-NEXT: s_mov_b64 s[8:9], s[2:3] +; GFX900-NEXT: s_mov_b64 s[6:7], 0 +; GFX900-NEXT: s_cbranch_vccnz .LBB2_4 +; GFX900-NEXT: ; %bb.3: ; %if +; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1 +; GFX900-NEXT: v_cmp_gt_i32_e32 vcc, 11, v5 +; GFX900-NEXT: s_andn2_b64 s[8:9], s[2:3], exec +; GFX900-NEXT: s_and_b64 s[12:13], vcc, exec +; GFX900-NEXT: s_mov_b64 s[6:7], -1 +; GFX900-NEXT: s_or_b64 s[8:9], s[8:9], s[12:13] +; GFX900-NEXT: .LBB2_4: ; %Flow +; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1 +; GFX900-NEXT: s_and_saveexec_b64 s[12:13], s[8:9] +; GFX900-NEXT: s_xor_b64 s[8:9], exec, s[12:13] +; GFX900-NEXT: s_cbranch_execz .LBB2_6 +; GFX900-NEXT: ; %bb.5: ; %else +; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1 +; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_add_u32_e32 v3, v3, v4 +; GFX900-NEXT: s_andn2_b64 s[6:7], s[6:7], exec +; GFX900-NEXT: .LBB2_6: ; %Flow1 +; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1 +; GFX900-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX900-NEXT: s_and_saveexec_b64 s[8:9], s[6:7] +; GFX900-NEXT: s_cbranch_execz .LBB2_1 +; GFX900-NEXT: ; %bb.7: ; %then +; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1 +; GFX900-NEXT: flat_store_dword v[1:2], v0 +; GFX900-NEXT: s_branch .LBB2_1 +; GFX900-NEXT: .LBB2_8: ; %end +; GFX900-NEXT: s_endpgm +entry: + %a = tail call i32 @llvm.amdgcn.workitem.id.x() + br label %loop + +loop: + %entry_phi = phi i32 [ 0, %entry ], [ %a15, %latch ] + %load = load %pair, ptr %ptr + br i1 %cond, label %if, label %else + +if: + %cmp = icmp sgt i32 %entry_phi, 10 + br i1 %cmp, label %then, label %else + +then: + %a_then = extractvalue %pair %load, 0 + store i32 %a, ptr %ptr, align 4 + br label %latch + +else: + %a2 = extractvalue %pair %load, 1 + %y = extractvalue %pair %load, 0 + %a_else = add i32 %y, %a2 + br label %latch + +latch: + %a_test = phi i32 [ %a_then, %then ], [ %a_else, %else ] + store i32 %a_test, ptr %ptr + %a15 = add nsw i32 %a_test, 20 + %a16 = icmp slt i32 %a15, 255 + br i1 %a16, label %loop, label %end + +end: + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll index d23e314..f6c357d 100644 --- a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll +++ b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll @@ -70,12 +70,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS1-NEXT: s_xor_b64 s[4:5], s[4:5], -1 ; GLOBALNESS1-NEXT: s_mov_b64 s[38:39], s[8:9] ; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[8:9], 1, v1 -; GLOBALNESS1-NEXT: ; implicit-def: $vgpr59 : SGPR spill to VGPR lane +; GLOBALNESS1-NEXT: ; implicit-def: $vgpr57 : SGPR spill to VGPR lane ; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[66:67], 1, v0 ; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5] -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s8, 0 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s8, 0 ; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[68:69], 1, v0 -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s9, 1 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s9, 1 ; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[70:71], 1, v3 ; GLOBALNESS1-NEXT: v_mov_b32_e32 v46, 0x80 ; GLOBALNESS1-NEXT: s_mov_b32 s82, s16 @@ -84,7 +84,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS1-NEXT: s_mov_b64 s[34:35], s[10:11] ; GLOBALNESS1-NEXT: v_mov_b32_e32 v47, 0 ; GLOBALNESS1-NEXT: s_mov_b32 s32, 0 -; GLOBALNESS1-NEXT: ; implicit-def: $vgpr56_vgpr57 +; GLOBALNESS1-NEXT: ; implicit-def: $vgpr58_vgpr59 ; GLOBALNESS1-NEXT: s_waitcnt vmcnt(0) ; GLOBALNESS1-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2 ; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc @@ -93,24 +93,24 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS1-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 ; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v0 ; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s4, 2 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s4, 2 ; GLOBALNESS1-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s5, 3 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s5, 3 ; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v3 ; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s4, 4 -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s5, 5 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s4, 4 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s5, 5 ; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v2 -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s4, 6 -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s5, 7 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s4, 6 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s5, 7 ; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[80:81], 1, v1 -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s70, 8 -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s71, 9 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s70, 8 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s71, 9 ; GLOBALNESS1-NEXT: s_branch .LBB1_4 ; GLOBALNESS1-NEXT: .LBB1_1: ; %bb70.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: v_readlane_b32 s6, v59, 6 -; GLOBALNESS1-NEXT: v_readlane_b32 s7, v59, 7 +; GLOBALNESS1-NEXT: v_readlane_b32 s6, v57, 6 +; GLOBALNESS1-NEXT: v_readlane_b32 s7, v57, 7 ; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[6:7] ; GLOBALNESS1-NEXT: s_cbranch_vccz .LBB1_28 ; GLOBALNESS1-NEXT: .LBB1_2: ; %Flow15 @@ -120,7 +120,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS1-NEXT: .LBB1_3: ; %Flow28 ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[6:7] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[56:57], v[0:1], v[0:1] op_sel:[0,1] +; GLOBALNESS1-NEXT: v_pk_mov_b32 v[58:59], v[0:1], v[0:1] op_sel:[0,1] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_29 ; GLOBALNESS1-NEXT: .LBB1_4: ; %bb5 ; GLOBALNESS1-NEXT: ; =>This Loop Header: Depth=1 @@ -128,7 +128,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS1-NEXT: flat_load_dword v40, v[46:47] ; GLOBALNESS1-NEXT: s_add_u32 s8, s38, 40 ; GLOBALNESS1-NEXT: buffer_store_dword v42, off, s[0:3], 0 -; GLOBALNESS1-NEXT: flat_load_dword v58, v[46:47] +; GLOBALNESS1-NEXT: flat_load_dword v56, v[46:47] ; GLOBALNESS1-NEXT: s_addc_u32 s9, s39, 0 ; GLOBALNESS1-NEXT: s_getpc_b64 s[4:5] ; GLOBALNESS1-NEXT: s_add_u32 s4, s4, wobble@gotpcrel32@lo+4 @@ -186,10 +186,10 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS1-NEXT: ; %bb.11: ; %bb33.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS1-NEXT: global_load_dwordx2 v[0:1], v[44:45], off -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s8, 10 -; GLOBALNESS1-NEXT: v_writelane_b32 v59, s9, 11 -; GLOBALNESS1-NEXT: v_readlane_b32 s4, v59, 2 -; GLOBALNESS1-NEXT: v_readlane_b32 s5, v59, 3 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s8, 10 +; GLOBALNESS1-NEXT: v_writelane_b32 v57, s9, 11 +; GLOBALNESS1-NEXT: v_readlane_b32 s4, v57, 2 +; GLOBALNESS1-NEXT: v_readlane_b32 s5, v57, 3 ; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[4:5] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_13 ; GLOBALNESS1-NEXT: ; %bb.12: ; %bb39.i @@ -198,7 +198,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[42:43], off ; GLOBALNESS1-NEXT: .LBB1_13: ; %bb44.lr.ph.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: v_cmp_ne_u32_e32 vcc, 0, v58 +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e32 vcc, 0, v56 ; GLOBALNESS1-NEXT: v_cndmask_b32_e32 v2, 0, v40, vcc ; GLOBALNESS1-NEXT: s_waitcnt vmcnt(0) ; GLOBALNESS1-NEXT: v_cmp_nlt_f64_e32 vcc, 0, v[0:1] @@ -228,8 +228,8 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_21 ; GLOBALNESS1-NEXT: ; %bb.19: ; %bb3.i.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2 -; GLOBALNESS1-NEXT: v_readlane_b32 s4, v59, 0 -; GLOBALNESS1-NEXT: v_readlane_b32 s5, v59, 1 +; GLOBALNESS1-NEXT: v_readlane_b32 s4, v57, 0 +; GLOBALNESS1-NEXT: v_readlane_b32 s5, v57, 1 ; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[4:5] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_21 ; GLOBALNESS1-NEXT: ; %bb.20: ; %bb6.i.i @@ -265,7 +265,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS1-NEXT: s_mov_b32 s13, s83 ; GLOBALNESS1-NEXT: s_mov_b32 s14, s82 ; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v41 -; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[56:57], off +; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[58:59], off ; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[54:55] ; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[96:97] ; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_14 @@ -277,13 +277,13 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS1-NEXT: .LBB1_24: ; %Flow23 ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS1-NEXT: s_load_dwordx4 s[4:7], s[38:39], 0x0 -; GLOBALNESS1-NEXT: v_readlane_b32 s70, v59, 8 -; GLOBALNESS1-NEXT: v_readlane_b32 s8, v59, 10 +; GLOBALNESS1-NEXT: v_readlane_b32 s70, v57, 8 +; GLOBALNESS1-NEXT: v_readlane_b32 s8, v57, 10 ; GLOBALNESS1-NEXT: v_pk_mov_b32 v[0:1], 0, 0 -; GLOBALNESS1-NEXT: v_readlane_b32 s71, v59, 9 +; GLOBALNESS1-NEXT: v_readlane_b32 s71, v57, 9 ; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0) ; GLOBALNESS1-NEXT: s_mov_b32 s55, s7 -; GLOBALNESS1-NEXT: v_readlane_b32 s9, v59, 11 +; GLOBALNESS1-NEXT: v_readlane_b32 s9, v57, 11 ; GLOBALNESS1-NEXT: .LBB1_25: ; %Flow24 ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[52:53] @@ -291,8 +291,8 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_2 ; GLOBALNESS1-NEXT: ; %bb.26: ; %bb67.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: v_readlane_b32 s6, v59, 4 -; GLOBALNESS1-NEXT: v_readlane_b32 s7, v59, 5 +; GLOBALNESS1-NEXT: v_readlane_b32 s6, v57, 4 +; GLOBALNESS1-NEXT: v_readlane_b32 s7, v57, 5 ; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[6:7] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_1 ; GLOBALNESS1-NEXT: ; %bb.27: ; %bb69.i @@ -384,12 +384,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS0-NEXT: s_xor_b64 s[4:5], s[4:5], -1 ; GLOBALNESS0-NEXT: s_mov_b64 s[38:39], s[8:9] ; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[8:9], 1, v1 -; GLOBALNESS0-NEXT: ; implicit-def: $vgpr59 : SGPR spill to VGPR lane +; GLOBALNESS0-NEXT: ; implicit-def: $vgpr57 : SGPR spill to VGPR lane ; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[66:67], 1, v0 ; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5] -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s8, 0 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s8, 0 ; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[68:69], 1, v0 -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s9, 1 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s9, 1 ; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[84:85], 1, v3 ; GLOBALNESS0-NEXT: v_mov_b32_e32 v46, 0x80 ; GLOBALNESS0-NEXT: s_mov_b32 s70, s16 @@ -398,7 +398,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS0-NEXT: s_mov_b64 s[34:35], s[10:11] ; GLOBALNESS0-NEXT: v_mov_b32_e32 v47, 0 ; GLOBALNESS0-NEXT: s_mov_b32 s32, 0 -; GLOBALNESS0-NEXT: ; implicit-def: $vgpr56_vgpr57 +; GLOBALNESS0-NEXT: ; implicit-def: $vgpr58_vgpr59 ; GLOBALNESS0-NEXT: s_waitcnt vmcnt(0) ; GLOBALNESS0-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2 ; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc @@ -407,24 +407,24 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS0-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 ; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v0 ; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s4, 2 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s4, 2 ; GLOBALNESS0-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s5, 3 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s5, 3 ; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v3 ; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s4, 4 -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s5, 5 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s4, 4 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s5, 5 ; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v2 -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s4, 6 -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s5, 7 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s4, 6 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s5, 7 ; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[80:81], 1, v1 -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s84, 8 -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s85, 9 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s84, 8 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s85, 9 ; GLOBALNESS0-NEXT: s_branch .LBB1_4 ; GLOBALNESS0-NEXT: .LBB1_1: ; %bb70.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: v_readlane_b32 s6, v59, 6 -; GLOBALNESS0-NEXT: v_readlane_b32 s7, v59, 7 +; GLOBALNESS0-NEXT: v_readlane_b32 s6, v57, 6 +; GLOBALNESS0-NEXT: v_readlane_b32 s7, v57, 7 ; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[6:7] ; GLOBALNESS0-NEXT: s_cbranch_vccz .LBB1_28 ; GLOBALNESS0-NEXT: .LBB1_2: ; %Flow15 @@ -434,7 +434,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS0-NEXT: .LBB1_3: ; %Flow28 ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[6:7] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[56:57], v[0:1], v[0:1] op_sel:[0,1] +; GLOBALNESS0-NEXT: v_pk_mov_b32 v[58:59], v[0:1], v[0:1] op_sel:[0,1] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_29 ; GLOBALNESS0-NEXT: .LBB1_4: ; %bb5 ; GLOBALNESS0-NEXT: ; =>This Loop Header: Depth=1 @@ -442,7 +442,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS0-NEXT: flat_load_dword v40, v[46:47] ; GLOBALNESS0-NEXT: s_add_u32 s8, s38, 40 ; GLOBALNESS0-NEXT: buffer_store_dword v42, off, s[0:3], 0 -; GLOBALNESS0-NEXT: flat_load_dword v58, v[46:47] +; GLOBALNESS0-NEXT: flat_load_dword v56, v[46:47] ; GLOBALNESS0-NEXT: s_addc_u32 s9, s39, 0 ; GLOBALNESS0-NEXT: s_getpc_b64 s[4:5] ; GLOBALNESS0-NEXT: s_add_u32 s4, s4, wobble@gotpcrel32@lo+4 @@ -500,10 +500,10 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS0-NEXT: ; %bb.11: ; %bb33.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS0-NEXT: global_load_dwordx2 v[0:1], v[44:45], off -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s8, 10 -; GLOBALNESS0-NEXT: v_writelane_b32 v59, s9, 11 -; GLOBALNESS0-NEXT: v_readlane_b32 s4, v59, 2 -; GLOBALNESS0-NEXT: v_readlane_b32 s5, v59, 3 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s8, 10 +; GLOBALNESS0-NEXT: v_writelane_b32 v57, s9, 11 +; GLOBALNESS0-NEXT: v_readlane_b32 s4, v57, 2 +; GLOBALNESS0-NEXT: v_readlane_b32 s5, v57, 3 ; GLOBALNESS0-NEXT: s_mov_b32 s83, s55 ; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[4:5] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_13 @@ -513,7 +513,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[42:43], off ; GLOBALNESS0-NEXT: .LBB1_13: ; %bb44.lr.ph.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: v_cmp_ne_u32_e32 vcc, 0, v58 +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e32 vcc, 0, v56 ; GLOBALNESS0-NEXT: v_cndmask_b32_e32 v2, 0, v40, vcc ; GLOBALNESS0-NEXT: s_waitcnt vmcnt(0) ; GLOBALNESS0-NEXT: v_cmp_nlt_f64_e32 vcc, 0, v[0:1] @@ -543,8 +543,8 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_21 ; GLOBALNESS0-NEXT: ; %bb.19: ; %bb3.i.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2 -; GLOBALNESS0-NEXT: v_readlane_b32 s4, v59, 0 -; GLOBALNESS0-NEXT: v_readlane_b32 s5, v59, 1 +; GLOBALNESS0-NEXT: v_readlane_b32 s4, v57, 0 +; GLOBALNESS0-NEXT: v_readlane_b32 s5, v57, 1 ; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[4:5] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_21 ; GLOBALNESS0-NEXT: ; %bb.20: ; %bb6.i.i @@ -580,7 +580,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS0-NEXT: s_mov_b32 s13, s71 ; GLOBALNESS0-NEXT: s_mov_b32 s14, s70 ; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v41 -; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[56:57], off +; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[58:59], off ; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[54:55] ; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[96:97] ; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_14 @@ -591,12 +591,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS0-NEXT: s_branch .LBB1_14 ; GLOBALNESS0-NEXT: .LBB1_24: ; %Flow23 ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: v_readlane_b32 s84, v59, 8 -; GLOBALNESS0-NEXT: v_readlane_b32 s8, v59, 10 +; GLOBALNESS0-NEXT: v_readlane_b32 s84, v57, 8 +; GLOBALNESS0-NEXT: v_readlane_b32 s8, v57, 10 ; GLOBALNESS0-NEXT: v_pk_mov_b32 v[0:1], 0, 0 ; GLOBALNESS0-NEXT: s_mov_b32 s55, s83 -; GLOBALNESS0-NEXT: v_readlane_b32 s85, v59, 9 -; GLOBALNESS0-NEXT: v_readlane_b32 s9, v59, 11 +; GLOBALNESS0-NEXT: v_readlane_b32 s85, v57, 9 +; GLOBALNESS0-NEXT: v_readlane_b32 s9, v57, 11 ; GLOBALNESS0-NEXT: .LBB1_25: ; %Flow24 ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[52:53] @@ -604,8 +604,8 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i ; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_2 ; GLOBALNESS0-NEXT: ; %bb.26: ; %bb67.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: v_readlane_b32 s6, v59, 4 -; GLOBALNESS0-NEXT: v_readlane_b32 s7, v59, 5 +; GLOBALNESS0-NEXT: v_readlane_b32 s6, v57, 4 +; GLOBALNESS0-NEXT: v_readlane_b32 s7, v57, 5 ; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[6:7] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_1 ; GLOBALNESS0-NEXT: ; %bb.27: ; %bb69.i diff --git a/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll b/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll index d0d1ba8..b3166fa 100644 --- a/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll +++ b/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll @@ -8,9 +8,8 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s17 ; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0 -; CHECK-NEXT: v_mov_b32_e32 v40, v0 -; CHECK-NEXT: v_pk_mov_b32 v[0:1], 0, 0 -; CHECK-NEXT: flat_load_dword v42, v[0:1] +; CHECK-NEXT: v_pk_mov_b32 v[44:45], 0, 0 +; CHECK-NEXT: flat_load_dword v42, v[44:45] ; CHECK-NEXT: s_mov_b64 s[38:39], s[6:7] ; CHECK-NEXT: s_mov_b64 s[48:49], s[4:5] ; CHECK-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x8 @@ -19,48 +18,44 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x ; CHECK-NEXT: s_addc_u32 s1, s1, 0 ; CHECK-NEXT: s_mov_b64 s[34:35], s[8:9] ; CHECK-NEXT: s_waitcnt lgkmcnt(0) -; CHECK-NEXT: v_mov_b32_e32 v46, s6 -; CHECK-NEXT: v_mov_b32_e32 v47, s7 +; CHECK-NEXT: v_accvgpr_write_b32 a32, s6 +; CHECK-NEXT: v_accvgpr_write_b32 a33, s7 ; CHECK-NEXT: s_mov_b64 s[6:7], src_private_base ; CHECK-NEXT: s_cmp_lg_u32 s64, -1 ; CHECK-NEXT: s_cselect_b32 s7, s7, 0 ; CHECK-NEXT: s_cselect_b32 s8, s64, 0 ; CHECK-NEXT: s_add_u32 s50, s34, 48 ; CHECK-NEXT: s_addc_u32 s51, s35, 0 -; CHECK-NEXT: v_pk_mov_b32 v[58:59], s[4:5], s[4:5] op_sel:[0,1] +; CHECK-NEXT: v_pk_mov_b32 v[56:57], s[4:5], s[4:5] op_sel:[0,1] ; CHECK-NEXT: s_getpc_b64 s[4:5] ; CHECK-NEXT: s_add_u32 s4, s4, G@gotpcrel32@lo+4 ; CHECK-NEXT: s_addc_u32 s5, s5, G@gotpcrel32@hi+12 ; CHECK-NEXT: s_load_dwordx2 s[54:55], s[4:5], 0x0 ; CHECK-NEXT: s_mov_b32 s6, 0 -; CHECK-NEXT: v_pk_mov_b32 v[0:1], 0, 0 -; CHECK-NEXT: v_mov_b32_e32 v57, s7 +; CHECK-NEXT: v_mov_b32_e32 v47, s7 ; CHECK-NEXT: s_mov_b32 s7, s6 ; CHECK-NEXT: s_mov_b32 s53, s14 -; CHECK-NEXT: v_accvgpr_write_b32 a33, v1 -; CHECK-NEXT: v_mov_b32_e32 v56, s8 -; CHECK-NEXT: v_pk_mov_b32 v[60:61], s[6:7], s[6:7] op_sel:[0,1] +; CHECK-NEXT: v_mov_b32_e32 v46, s8 +; CHECK-NEXT: v_pk_mov_b32 v[58:59], s[6:7], s[6:7] op_sel:[0,1] ; CHECK-NEXT: s_mov_b64 s[4:5], s[48:49] ; CHECK-NEXT: s_mov_b64 s[6:7], s[38:39] ; CHECK-NEXT: s_mov_b64 s[8:9], s[50:51] ; CHECK-NEXT: s_mov_b32 s12, s14 ; CHECK-NEXT: s_mov_b32 s13, s15 ; CHECK-NEXT: s_mov_b32 s14, s16 -; CHECK-NEXT: v_mov_b32_e32 v31, v40 +; CHECK-NEXT: v_mov_b32_e32 v31, v0 ; CHECK-NEXT: s_mov_b32 s32, 0 ; CHECK-NEXT: s_mov_b32 s33, s16 ; CHECK-NEXT: s_mov_b32 s52, s15 ; CHECK-NEXT: s_mov_b64 s[36:37], s[10:11] -; CHECK-NEXT: v_accvgpr_write_b32 a32, v0 -; CHECK-NEXT: flat_store_dwordx2 v[58:59], v[60:61] +; CHECK-NEXT: v_mov_b32_e32 v40, v0 +; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[58:59] ; CHECK-NEXT: ; kill: def $sgpr15 killed $sgpr15 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_swappc_b64 s[30:31], s[54:55] -; CHECK-NEXT: flat_load_dwordx2 v[62:63], v[58:59] -; CHECK-NEXT: v_accvgpr_read_b32 v0, a32 -; CHECK-NEXT: v_mov_b32_e32 v44, 0 -; CHECK-NEXT: v_mov_b32_e32 v45, 0x3ff00000 -; CHECK-NEXT: v_accvgpr_read_b32 v1, a33 +; CHECK-NEXT: flat_load_dwordx2 v[60:61], v[56:57] +; CHECK-NEXT: v_mov_b32_e32 v62, 0 +; CHECK-NEXT: v_mov_b32_e32 v63, 0x3ff00000 ; CHECK-NEXT: s_mov_b64 s[4:5], s[48:49] ; CHECK-NEXT: s_mov_b64 s[6:7], s[38:39] ; CHECK-NEXT: s_mov_b64 s[8:9], s[50:51] @@ -69,20 +64,20 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x ; CHECK-NEXT: s_mov_b32 s13, s52 ; CHECK-NEXT: s_mov_b32 s14, s33 ; CHECK-NEXT: v_mov_b32_e32 v31, v40 -; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[44:45] -; CHECK-NEXT: flat_store_dwordx2 v[58:59], v[60:61] +; CHECK-NEXT: flat_store_dwordx2 v[44:45], v[62:63] +; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[58:59] ; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: ; kill: def $sgpr15 killed $sgpr15 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[54:55] -; CHECK-NEXT: flat_load_dwordx2 v[0:1], v[56:57] glc +; CHECK-NEXT: flat_load_dwordx2 v[0:1], v[46:47] glc ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; CHECK-NEXT: v_mov_b32_e32 v0, s64 ; CHECK-NEXT: v_cmp_lt_i32_e32 vcc, 0, v42 -; CHECK-NEXT: flat_store_dwordx2 v[58:59], v[62:63] +; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[60:61] ; CHECK-NEXT: s_waitcnt vmcnt(0) -; CHECK-NEXT: flat_store_dwordx2 v[58:59], v[46:47] -; CHECK-NEXT: buffer_store_dword v47, v0, s[0:3], 0 offen offset:4 -; CHECK-NEXT: buffer_store_dword v44, v0, s[0:3], 0 offen +; CHECK-NEXT: flat_store_dwordx2 v[56:57], a[32:33] +; CHECK-NEXT: buffer_store_dword a33, v0, s[0:3], 0 offen offset:4 +; CHECK-NEXT: buffer_store_dword v62, v0, s[0:3], 0 offen ; CHECK-NEXT: ; implicit-def: $vgpr4 ; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc ; CHECK-NEXT: s_xor_b64 s[4:5], exec, s[4:5] diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll index 2f25a93..fe7def8a 100644 --- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll +++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll @@ -1961,16 +1961,15 @@ define <6 x half> @shuffle_v6f16_452367(ptr addrspace(1) %arg0, ptr addrspace(1) ; GFX942-LABEL: shuffle_v6f16_452367: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX942-NEXT: v_mov_b32_e32 v7, v1 -; GFX942-NEXT: v_mov_b32_e32 v6, v0 -; GFX942-NEXT: v_mov_b32_e32 v5, v3 -; GFX942-NEXT: v_mov_b32_e32 v4, v2 -; GFX942-NEXT: global_load_dwordx3 v[0:2], v[6:7], off -; GFX942-NEXT: global_load_dword v3, v[4:5], off +; GFX942-NEXT: global_load_dwordx3 v[4:6], v[0:1], off +; GFX942-NEXT: global_load_dword v4, v[2:3], off +; GFX942-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GFX942-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX942-NEXT: s_waitcnt vmcnt(1) -; GFX942-NEXT: v_mov_b32_e32 v0, v2 +; GFX942-NEXT: v_mov_b32_e32 v0, v6 +; GFX942-NEXT: v_mov_b32_e32 v1, v5 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: v_mov_b32_e32 v2, v3 +; GFX942-NEXT: v_mov_b32_e32 v2, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v6f16_452367: @@ -5151,16 +5150,15 @@ define <6 x bfloat> @shuffle_v6bf16_452367(ptr addrspace(1) %arg0, ptr addrspace ; GFX942-LABEL: shuffle_v6bf16_452367: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX942-NEXT: v_mov_b32_e32 v7, v1 -; GFX942-NEXT: v_mov_b32_e32 v6, v0 -; GFX942-NEXT: v_mov_b32_e32 v5, v3 -; GFX942-NEXT: v_mov_b32_e32 v4, v2 -; GFX942-NEXT: global_load_dwordx3 v[0:2], v[6:7], off -; GFX942-NEXT: global_load_dword v3, v[4:5], off +; GFX942-NEXT: global_load_dwordx3 v[4:6], v[0:1], off +; GFX942-NEXT: global_load_dword v4, v[2:3], off +; GFX942-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GFX942-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX942-NEXT: s_waitcnt vmcnt(1) -; GFX942-NEXT: v_mov_b32_e32 v0, v2 +; GFX942-NEXT: v_mov_b32_e32 v0, v6 +; GFX942-NEXT: v_mov_b32_e32 v1, v5 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: v_mov_b32_e32 v2, v3 +; GFX942-NEXT: v_mov_b32_e32 v2, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v6bf16_452367: diff --git a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll index a401f989..d8264b5a 100644 --- a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll +++ b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll @@ -58,19 +58,19 @@ define amdgpu_kernel void @v4i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 ; GFX942-NEXT: v_and_b32_e32 v3, 0x3ff, v0 -; GFX942-NEXT: v_lshlrev_b32_e32 v2, 2, v3 +; GFX942-NEXT: v_lshlrev_b32_e32 v1, 2, v3 ; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_load_dword v1, v2, s[0:1] +; GFX942-NEXT: global_load_dword v2, v1, s[0:1] ; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v3 ; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX942-NEXT: s_cbranch_execz .LBB1_2 ; GFX942-NEXT: ; %bb.1: ; %bb.1 -; GFX942-NEXT: global_load_dword v1, v2, s[2:3] +; GFX942-NEXT: global_load_dword v2, v1, s[2:3] ; GFX942-NEXT: .LBB1_2: ; %bb.2 ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dword v0, v1, s[6:7] +; GFX942-NEXT: global_store_dword v0, v2, s[6:7] ; GFX942-NEXT: s_endpgm entry: %idx = call i32 @llvm.amdgcn.workitem.id.x() @@ -136,19 +136,19 @@ define amdgpu_kernel void @v8i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 ; GFX942-NEXT: v_and_b32_e32 v4, 0x3ff, v0 -; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v4 -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v4 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[0:1] +; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[0:1] ; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v4 ; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX942-NEXT: s_cbranch_execz .LBB3_2 ; GFX942-NEXT: ; %bb.1: ; %bb.1 -; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[2:3] +; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[2:3] ; GFX942-NEXT: .LBB3_2: ; %bb.2 ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[6:7] ; GFX942-NEXT: s_endpgm entry: %idx = call i32 @llvm.amdgcn.workitem.id.x() @@ -173,19 +173,19 @@ define amdgpu_kernel void @v16i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1 ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 ; GFX942-NEXT: v_and_b32_e32 v6, 0x3ff, v0 -; GFX942-NEXT: v_lshlrev_b32_e32 v5, 4, v6 -; GFX942-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-NEXT: v_lshlrev_b32_e32 v1, 4, v6 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_load_dwordx4 v[0:3], v5, s[0:1] +; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1] ; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v6 ; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX942-NEXT: s_cbranch_execz .LBB4_2 ; GFX942-NEXT: ; %bb.1: ; %bb.1 -; GFX942-NEXT: global_load_dwordx4 v[0:3], v5, s[2:3] +; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[2:3] ; GFX942-NEXT: .LBB4_2: ; %bb.2 ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[6:7] ; GFX942-NEXT: s_endpgm entry: %idx = call i32 @llvm.amdgcn.workitem.id.x() @@ -210,23 +210,23 @@ define amdgpu_kernel void @v32i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1 ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 ; GFX942-NEXT: v_and_b32_e32 v10, 0x3ff, v0 -; GFX942-NEXT: v_lshlrev_b32_e32 v9, 5, v10 -; GFX942-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-NEXT: v_lshlrev_b32_e32 v1, 5, v10 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_load_dwordx4 v[4:7], v9, s[0:1] offset:16 -; GFX942-NEXT: global_load_dwordx4 v[0:3], v9, s[0:1] +; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[0:1] offset:16 +; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1] ; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v10 ; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX942-NEXT: s_cbranch_execz .LBB5_2 ; GFX942-NEXT: ; %bb.1: ; %bb.1 -; GFX942-NEXT: global_load_dwordx4 v[4:7], v9, s[2:3] offset:16 -; GFX942-NEXT: global_load_dwordx4 v[0:3], v9, s[2:3] +; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[2:3] offset:16 +; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[2:3] ; GFX942-NEXT: .LBB5_2: ; %bb.2 ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_waitcnt vmcnt(1) -; GFX942-NEXT: global_store_dwordx4 v8, v[4:7], s[6:7] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] offset:16 ; GFX942-NEXT: s_waitcnt vmcnt(1) -; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[6:7] +; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[6:7] ; GFX942-NEXT: s_endpgm entry: %idx = call i32 @llvm.amdgcn.workitem.id.x() @@ -250,72 +250,72 @@ define amdgpu_kernel void @v256i8_liveout(ptr addrspace(1) %src1, ptr addrspace( ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 -; GFX942-NEXT: v_and_b32_e32 v2, 0x3ff, v0 -; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v2 +; GFX942-NEXT: v_and_b32_e32 v62, 0x3ff, v0 +; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v62 ; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_load_dwordx4 v[28:31], v1, s[0:1] offset:240 -; GFX942-NEXT: global_load_dwordx4 v[24:27], v1, s[0:1] offset:224 -; GFX942-NEXT: global_load_dwordx4 v[20:23], v1, s[0:1] offset:208 -; GFX942-NEXT: global_load_dwordx4 v[16:19], v1, s[0:1] offset:192 -; GFX942-NEXT: global_load_dwordx4 v[12:15], v1, s[0:1] offset:176 -; GFX942-NEXT: global_load_dwordx4 v[8:11], v1, s[0:1] offset:160 -; GFX942-NEXT: global_load_dwordx4 v[4:7], v1, s[0:1] offset:144 -; GFX942-NEXT: global_load_dwordx4 a[0:3], v1, s[0:1] offset:128 -; GFX942-NEXT: global_load_dwordx4 v[60:63], v1, s[0:1] offset:112 -; GFX942-NEXT: global_load_dwordx4 v[56:59], v1, s[0:1] offset:96 -; GFX942-NEXT: global_load_dwordx4 v[52:55], v1, s[0:1] offset:80 -; GFX942-NEXT: global_load_dwordx4 v[48:51], v1, s[0:1] offset:64 -; GFX942-NEXT: global_load_dwordx4 v[44:47], v1, s[0:1] offset:48 -; GFX942-NEXT: global_load_dwordx4 v[40:43], v1, s[0:1] offset:32 -; GFX942-NEXT: global_load_dwordx4 v[36:39], v1, s[0:1] offset:16 -; GFX942-NEXT: global_load_dwordx4 v[32:35], v1, s[0:1] -; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v2 +; GFX942-NEXT: global_load_dwordx4 v[30:33], v1, s[0:1] offset:240 +; GFX942-NEXT: global_load_dwordx4 v[26:29], v1, s[0:1] offset:224 +; GFX942-NEXT: global_load_dwordx4 v[22:25], v1, s[0:1] offset:208 +; GFX942-NEXT: global_load_dwordx4 v[18:21], v1, s[0:1] offset:192 +; GFX942-NEXT: global_load_dwordx4 v[14:17], v1, s[0:1] offset:176 +; GFX942-NEXT: global_load_dwordx4 v[10:13], v1, s[0:1] offset:160 +; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[0:1] offset:144 +; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1] offset:128 +; GFX942-NEXT: global_load_dwordx4 a[0:3], v1, s[0:1] offset:112 +; GFX942-NEXT: global_load_dwordx4 v[58:61], v1, s[0:1] offset:96 +; GFX942-NEXT: global_load_dwordx4 v[54:57], v1, s[0:1] offset:80 +; GFX942-NEXT: global_load_dwordx4 v[50:53], v1, s[0:1] offset:64 +; GFX942-NEXT: global_load_dwordx4 v[46:49], v1, s[0:1] offset:48 +; GFX942-NEXT: global_load_dwordx4 v[42:45], v1, s[0:1] offset:32 +; GFX942-NEXT: global_load_dwordx4 v[38:41], v1, s[0:1] offset:16 +; GFX942-NEXT: global_load_dwordx4 v[34:37], v1, s[0:1] +; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v62 ; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX942-NEXT: s_cbranch_execz .LBB6_2 ; GFX942-NEXT: ; %bb.1: ; %bb.1 -; GFX942-NEXT: global_load_dwordx4 v[28:31], v1, s[2:3] offset:240 -; GFX942-NEXT: global_load_dwordx4 v[24:27], v1, s[2:3] offset:224 -; GFX942-NEXT: global_load_dwordx4 v[20:23], v1, s[2:3] offset:208 -; GFX942-NEXT: global_load_dwordx4 v[16:19], v1, s[2:3] offset:192 -; GFX942-NEXT: global_load_dwordx4 v[12:15], v1, s[2:3] offset:176 -; GFX942-NEXT: global_load_dwordx4 v[8:11], v1, s[2:3] offset:160 -; GFX942-NEXT: global_load_dwordx4 v[4:7], v1, s[2:3] offset:144 -; GFX942-NEXT: global_load_dwordx4 a[0:3], v1, s[2:3] offset:128 -; GFX942-NEXT: global_load_dwordx4 v[60:63], v1, s[2:3] offset:112 -; GFX942-NEXT: global_load_dwordx4 v[56:59], v1, s[2:3] offset:96 -; GFX942-NEXT: global_load_dwordx4 v[52:55], v1, s[2:3] offset:80 -; GFX942-NEXT: global_load_dwordx4 v[48:51], v1, s[2:3] offset:64 -; GFX942-NEXT: global_load_dwordx4 v[44:47], v1, s[2:3] offset:48 -; GFX942-NEXT: global_load_dwordx4 v[40:43], v1, s[2:3] offset:32 -; GFX942-NEXT: global_load_dwordx4 v[36:39], v1, s[2:3] offset:16 -; GFX942-NEXT: global_load_dwordx4 v[32:35], v1, s[2:3] +; GFX942-NEXT: global_load_dwordx4 v[30:33], v1, s[2:3] offset:240 +; GFX942-NEXT: global_load_dwordx4 v[26:29], v1, s[2:3] offset:224 +; GFX942-NEXT: global_load_dwordx4 v[22:25], v1, s[2:3] offset:208 +; GFX942-NEXT: global_load_dwordx4 v[18:21], v1, s[2:3] offset:192 +; GFX942-NEXT: global_load_dwordx4 v[14:17], v1, s[2:3] offset:176 +; GFX942-NEXT: global_load_dwordx4 v[10:13], v1, s[2:3] offset:160 +; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[2:3] offset:144 +; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[2:3] offset:128 +; GFX942-NEXT: global_load_dwordx4 a[0:3], v1, s[2:3] offset:112 +; GFX942-NEXT: global_load_dwordx4 v[58:61], v1, s[2:3] offset:96 +; GFX942-NEXT: global_load_dwordx4 v[54:57], v1, s[2:3] offset:80 +; GFX942-NEXT: global_load_dwordx4 v[50:53], v1, s[2:3] offset:64 +; GFX942-NEXT: global_load_dwordx4 v[46:49], v1, s[2:3] offset:48 +; GFX942-NEXT: global_load_dwordx4 v[42:45], v1, s[2:3] offset:32 +; GFX942-NEXT: global_load_dwordx4 v[38:41], v1, s[2:3] offset:16 +; GFX942-NEXT: global_load_dwordx4 v[34:37], v1, s[2:3] ; GFX942-NEXT: .LBB6_2: ; %bb.2 ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_waitcnt vmcnt(7) -; GFX942-NEXT: global_store_dwordx4 v0, v[60:63], s[6:7] offset:112 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] offset:112 ; GFX942-NEXT: s_waitcnt vmcnt(7) -; GFX942-NEXT: global_store_dwordx4 v0, v[56:59], s[6:7] offset:96 +; GFX942-NEXT: global_store_dwordx4 v0, v[58:61], s[6:7] offset:96 ; GFX942-NEXT: s_waitcnt vmcnt(7) -; GFX942-NEXT: global_store_dwordx4 v0, v[52:55], s[6:7] offset:80 +; GFX942-NEXT: global_store_dwordx4 v0, v[54:57], s[6:7] offset:80 ; GFX942-NEXT: s_waitcnt vmcnt(7) -; GFX942-NEXT: global_store_dwordx4 v0, v[48:51], s[6:7] offset:64 +; GFX942-NEXT: global_store_dwordx4 v0, v[50:53], s[6:7] offset:64 ; GFX942-NEXT: s_waitcnt vmcnt(7) -; GFX942-NEXT: global_store_dwordx4 v0, v[44:47], s[6:7] offset:48 +; GFX942-NEXT: global_store_dwordx4 v0, v[46:49], s[6:7] offset:48 ; GFX942-NEXT: s_waitcnt vmcnt(7) -; GFX942-NEXT: global_store_dwordx4 v0, v[40:43], s[6:7] offset:32 +; GFX942-NEXT: global_store_dwordx4 v0, v[42:45], s[6:7] offset:32 ; GFX942-NEXT: s_waitcnt vmcnt(7) -; GFX942-NEXT: global_store_dwordx4 v0, v[36:39], s[6:7] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, v[38:41], s[6:7] offset:16 ; GFX942-NEXT: s_waitcnt vmcnt(7) -; GFX942-NEXT: global_store_dwordx4 v0, v[32:35], s[6:7] -; GFX942-NEXT: global_store_dwordx4 v0, v[28:31], s[6:7] offset:240 -; GFX942-NEXT: global_store_dwordx4 v0, v[24:27], s[6:7] offset:224 -; GFX942-NEXT: global_store_dwordx4 v0, v[20:23], s[6:7] offset:208 -; GFX942-NEXT: global_store_dwordx4 v0, v[16:19], s[6:7] offset:192 -; GFX942-NEXT: global_store_dwordx4 v0, v[12:15], s[6:7] offset:176 -; GFX942-NEXT: global_store_dwordx4 v0, v[8:11], s[6:7] offset:160 -; GFX942-NEXT: global_store_dwordx4 v0, v[4:7], s[6:7] offset:144 -; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] offset:128 +; GFX942-NEXT: global_store_dwordx4 v0, v[34:37], s[6:7] +; GFX942-NEXT: global_store_dwordx4 v0, v[30:33], s[6:7] offset:240 +; GFX942-NEXT: global_store_dwordx4 v0, v[26:29], s[6:7] offset:224 +; GFX942-NEXT: global_store_dwordx4 v0, v[22:25], s[6:7] offset:208 +; GFX942-NEXT: global_store_dwordx4 v0, v[18:21], s[6:7] offset:192 +; GFX942-NEXT: global_store_dwordx4 v0, v[14:17], s[6:7] offset:176 +; GFX942-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7] offset:160 +; GFX942-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] offset:144 +; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[6:7] offset:128 ; GFX942-NEXT: s_endpgm entry: %idx = call i32 @llvm.amdgcn.workitem.id.x() @@ -391,17 +391,17 @@ define amdgpu_kernel void @v8i8_phi_chain(ptr addrspace(1) %src1, ptr addrspace( ; GFX942-LABEL: v8i8_phi_chain: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 -; GFX942-NEXT: v_and_b32_e32 v2, 0x3ff, v0 -; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v2 -; GFX942-NEXT: v_cmp_lt_u32_e64 s[0:1], 14, v2 -; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v2 +; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v0 +; GFX942-NEXT: v_cmp_lt_u32_e64 s[0:1], 14, v0 +; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[8:9] +; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[8:9] ; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX942-NEXT: s_cbranch_execz .LBB8_2 ; GFX942-NEXT: ; %bb.1: ; %bb.1 -; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[10:11] -; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v2 +; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[10:11] +; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v0 ; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], exec ; GFX942-NEXT: s_and_b64 s[4:5], vcc, exec ; GFX942-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5] @@ -410,14 +410,14 @@ define amdgpu_kernel void @v8i8_phi_chain(ptr addrspace(1) %src1, ptr addrspace( ; GFX942-NEXT: s_and_saveexec_b64 s[2:3], s[0:1] ; GFX942-NEXT: s_cbranch_execz .LBB8_4 ; GFX942-NEXT: ; %bb.3: ; %bb.2 -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[12:13] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[12:13] ; GFX942-NEXT: .LBB8_4: ; %bb.3 ; GFX942-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[14:15] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[14:15] ; GFX942-NEXT: s_endpgm entry: %idx = call i32 @llvm.amdgcn.workitem.id.x() @@ -447,38 +447,38 @@ define amdgpu_kernel void @v8i8_phi_zeroinit(ptr addrspace(1) %src1, ptr addrspa ; GFX942-LABEL: v8i8_phi_zeroinit: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 -; GFX942-NEXT: v_and_b32_e32 v4, 0x3ff, v0 -; GFX942-NEXT: v_lshlrev_b32_e32 v5, 3, v4 -; GFX942-NEXT: v_cmp_lt_u32_e64 s[0:1], 14, v4 -; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v4 +; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v0 +; GFX942-NEXT: v_cmp_lt_u32_e64 s[0:1], 14, v0 +; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_load_dwordx2 v[0:1], v5, s[8:9] -; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[8:9] +; GFX942-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX942-NEXT: s_cbranch_execz .LBB9_2 ; GFX942-NEXT: ; %bb.1: ; %bb.1 -; GFX942-NEXT: global_load_dwordx2 v[2:3], v5, s[10:11] -; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v4 +; GFX942-NEXT: global_load_dwordx2 v[4:5], v1, s[10:11] +; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v0 ; GFX942-NEXT: s_waitcnt vmcnt(1) -; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 0 ; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], exec ; GFX942-NEXT: s_and_b64 s[4:5], vcc, exec -; GFX942-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-NEXT: v_mov_b32_e32 v3, v2 ; GFX942-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5] ; GFX942-NEXT: .LBB9_2: ; %Flow ; GFX942-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX942-NEXT: s_and_saveexec_b64 s[2:3], s[0:1] ; GFX942-NEXT: s_cbranch_execz .LBB9_4 ; GFX942-NEXT: ; %bb.3: ; %bb.2 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: v_mov_b32_e32 v2, 0 -; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[12:13] -; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[0:1] +; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3] +; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[12:13] ; GFX942-NEXT: .LBB9_4: ; %bb.3 ; GFX942-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v0, 0 -; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[14:15] +; GFX942-NEXT: s_waitcnt vmcnt(0) +; GFX942-NEXT: global_store_dwordx2 v0, v[4:5], s[14:15] ; GFX942-NEXT: s_endpgm entry: %idx = call i32 @llvm.amdgcn.workitem.id.x() @@ -617,30 +617,30 @@ define amdgpu_kernel void @v8i8_multi_block(ptr addrspace(1) %src1, ptr addrspac ; GFX942-LABEL: v8i8_multi_block: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 -; GFX942-NEXT: v_and_b32_e32 v5, 0x3ff, v0 -; GFX942-NEXT: v_lshlrev_b32_e32 v6, 3, v5 -; GFX942-NEXT: v_mov_b32_e32 v4, 0 -; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v5 +; GFX942-NEXT: v_and_b32_e32 v3, 0x3ff, v0 +; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v3 +; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v3 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_load_dwordx2 v[2:3], v6, s[8:9] +; GFX942-NEXT: global_load_dwordx2 v[0:1], v4, s[8:9] ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[0:1] ; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX942-NEXT: s_cbranch_execz .LBB11_4 ; GFX942-NEXT: ; %bb.1: ; %bb.1 -; GFX942-NEXT: global_load_dwordx2 v[0:1], v6, s[10:11] -; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v5 +; GFX942-NEXT: global_load_dwordx2 v[6:7], v4, s[10:11] +; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v3 ; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX942-NEXT: s_cbranch_execz .LBB11_3 ; GFX942-NEXT: ; %bb.2: ; %bb.2 -; GFX942-NEXT: v_mov_b32_e32 v5, 0 -; GFX942-NEXT: global_store_dwordx2 v5, v[2:3], s[12:13] +; GFX942-NEXT: v_mov_b32_e32 v3, 0 +; GFX942-NEXT: global_store_dwordx2 v3, v[0:1], s[12:13] ; GFX942-NEXT: .LBB11_3: ; %Flow ; GFX942-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX942-NEXT: .LBB11_4: ; %bb.3 ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_waitcnt vmcnt(0) -; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[14:15] +; GFX942-NEXT: global_store_dwordx2 v2, v[6:7], s[14:15] ; GFX942-NEXT: s_endpgm entry: %idx = call i32 @llvm.amdgcn.workitem.id.x() @@ -859,15 +859,15 @@ define amdgpu_kernel void @v8i8_mfma_i8(ptr addrspace(1) %src1, ptr addrspace(1) ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX942-NEXT: v_and_b32_e32 v4, 0x3ff, v0 -; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v4 -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v4 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v4 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[8:9] +; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[8:9] ; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX942-NEXT: s_cbranch_execz .LBB14_2 ; GFX942-NEXT: ; %bb.1: ; %bb.1 -; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[10:11] +; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[10:11] ; GFX942-NEXT: .LBB14_2: ; %bb.2 ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[14:15], 0x0 @@ -878,9 +878,9 @@ define amdgpu_kernel void @v8i8_mfma_i8(ptr addrspace(1) %src1, ptr addrspace(1) ; GFX942-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_nop 0 -; GFX942-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[2:3], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-NEXT: s_nop 6 -; GFX942-NEXT: global_store_dwordx4 v2, a[0:3], s[12:13] +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[12:13] ; GFX942-NEXT: s_endpgm entry: %idx = call i32 @llvm.amdgcn.workitem.id.x() @@ -909,15 +909,15 @@ define amdgpu_kernel void @v8i8_mfma_half(ptr addrspace(1) %src1, ptr addrspace( ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx8 s[36:43], s[4:5], 0x24 ; GFX942-NEXT: v_and_b32_e32 v4, 0x3ff, v0 -; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v4 -; GFX942-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v4 +; GFX942-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v4 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) -; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[36:37] +; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[36:37] ; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX942-NEXT: s_cbranch_execz .LBB15_2 ; GFX942-NEXT: ; %bb.1: ; %bb.1 -; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[38:39] +; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[38:39] ; GFX942-NEXT: .LBB15_2: ; %bb.2 ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_load_dwordx16 s[16:31], s[42:43], 0x0 @@ -957,18 +957,18 @@ define amdgpu_kernel void @v8i8_mfma_half(ptr addrspace(1) %src1, ptr addrspace( ; GFX942-NEXT: v_accvgpr_write_b32 a31, s15 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_nop 0 -; GFX942-NEXT: v_mfma_f32_32x32x4_2b_f16 a[0:31], v[0:1], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: v_mfma_f32_32x32x4_2b_f16 a[0:31], v[2:3], v[2:3], a[0:31] cbsz:1 abid:2 blgp:3 ; GFX942-NEXT: s_nop 7 ; GFX942-NEXT: s_nop 7 ; GFX942-NEXT: s_nop 2 -; GFX942-NEXT: global_store_dwordx4 v2, a[28:31], s[40:41] offset:112 -; GFX942-NEXT: global_store_dwordx4 v2, a[24:27], s[40:41] offset:96 -; GFX942-NEXT: global_store_dwordx4 v2, a[20:23], s[40:41] offset:80 -; GFX942-NEXT: global_store_dwordx4 v2, a[16:19], s[40:41] offset:64 -; GFX942-NEXT: global_store_dwordx4 v2, a[12:15], s[40:41] offset:48 -; GFX942-NEXT: global_store_dwordx4 v2, a[8:11], s[40:41] offset:32 -; GFX942-NEXT: global_store_dwordx4 v2, a[4:7], s[40:41] offset:16 -; GFX942-NEXT: global_store_dwordx4 v2, a[0:3], s[40:41] +; GFX942-NEXT: global_store_dwordx4 v0, a[28:31], s[40:41] offset:112 +; GFX942-NEXT: global_store_dwordx4 v0, a[24:27], s[40:41] offset:96 +; GFX942-NEXT: global_store_dwordx4 v0, a[20:23], s[40:41] offset:80 +; GFX942-NEXT: global_store_dwordx4 v0, a[16:19], s[40:41] offset:64 +; GFX942-NEXT: global_store_dwordx4 v0, a[12:15], s[40:41] offset:48 +; GFX942-NEXT: global_store_dwordx4 v0, a[8:11], s[40:41] offset:32 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[40:41] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[40:41] ; GFX942-NEXT: s_endpgm entry: %idx = call i32 @llvm.amdgcn.workitem.id.x() diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll index 145f1e4..ff18b32 100644 --- a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll +++ b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll @@ -2,7 +2,7 @@ ; A negative test to capture the expected error when the VGPRs are insufficient for wwm-regalloc. -; CHECK: error: can't find enough VGPRs for wwm-regalloc +; CHECK: error: cannot find enough VGPRs for wwm-regalloc define amdgpu_kernel void @test(i32 %in) { entry: diff --git a/llvm/test/CodeGen/DirectX/Binding/binding-overlap-6.ll b/llvm/test/CodeGen/DirectX/Binding/binding-overlap-6.ll new file mode 100644 index 0000000..3c37e63 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/Binding/binding-overlap-6.ll @@ -0,0 +1,24 @@ +; RUN: not opt -S -passes='dxil-post-optimization-validation' -mtriple=dxil-pc-shadermodel6.3-library %s 2>&1 | FileCheck %s + +; Check overlap with unbounded array + +; A overlaps with B +; RWBuffer<float> A[3] : register(u0); +; RWBuffer<float> B[] : register(u4); +; RWBuffer<float> C : register(u17); + +; CHECK: error: resource B at register 4 overlaps with resource C at register 17 in space 0 + +target triple = "dxil-pc-shadermodel6.3-library" + +@A.str = private unnamed_addr constant [2 x i8] c"A\00", align 1 +@B.str = private unnamed_addr constant [2 x i8] c"B\00", align 1 +@C.str = private unnamed_addr constant [2 x i8] c"C\00", align 1 + +define void @test_overlapping() { +entry: + %h1 = call target("dx.TypedBuffer", float, 1, 0, 0) @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 3, i32 0, i1 false, ptr @A.str) + %h2 = call target("dx.TypedBuffer", float, 1, 0, 0) @llvm.dx.resource.handlefrombinding(i32 0, i32 4, i32 -1, i32 0, i1 false, ptr @B.str) + %h3 = call target("dx.TypedBuffer", float, 1, 0, 0) @llvm.dx.resource.handlefrombinding(i32 0, i32 17, i32 1, i32 0, i1 false, ptr @C.str) + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll b/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll index 086ef54..c18c637 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll @@ -355,40 +355,46 @@ entry: define void @buildvector_v32i8_partial(ptr %dst, i8 %a0, i8 %a1, i8 %a2, i8 %a5, i8 %a7, i8 %a8, i8 %a15, i8 %a17, i8 %a18, i8 %a20, i8 %a22, i8 %a23, i8 %a27, i8 %a28, i8 %a31) nounwind { ; CHECK-LABEL: buildvector_v32i8_partial: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: addi.d $sp, $sp, -96 -; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill -; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill -; CHECK-NEXT: addi.d $fp, $sp, 96 -; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 -; CHECK-NEXT: ld.b $t0, $fp, 0 -; CHECK-NEXT: ld.b $t1, $fp, 8 -; CHECK-NEXT: ld.b $t2, $fp, 16 -; CHECK-NEXT: ld.b $t3, $fp, 24 -; CHECK-NEXT: ld.b $t4, $fp, 56 -; CHECK-NEXT: ld.b $t5, $fp, 48 -; CHECK-NEXT: ld.b $t6, $fp, 40 -; CHECK-NEXT: ld.b $t7, $fp, 32 -; CHECK-NEXT: st.b $t4, $sp, 63 -; CHECK-NEXT: st.b $t5, $sp, 60 -; CHECK-NEXT: st.b $t6, $sp, 59 -; CHECK-NEXT: st.b $t7, $sp, 55 -; CHECK-NEXT: st.b $t3, $sp, 54 -; CHECK-NEXT: st.b $t2, $sp, 52 -; CHECK-NEXT: st.b $t1, $sp, 50 -; CHECK-NEXT: st.b $t0, $sp, 49 -; CHECK-NEXT: st.b $a7, $sp, 47 -; CHECK-NEXT: st.b $a6, $sp, 40 -; CHECK-NEXT: st.b $a5, $sp, 39 -; CHECK-NEXT: st.b $a4, $sp, 37 -; CHECK-NEXT: st.b $a3, $sp, 34 -; CHECK-NEXT: st.b $a2, $sp, 33 -; CHECK-NEXT: st.b $a1, $sp, 32 -; CHECK-NEXT: xvld $xr0, $sp, 32 -; CHECK-NEXT: xvst $xr0, $a0, 0 -; CHECK-NEXT: addi.d $sp, $fp, -96 -; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload -; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload -; CHECK-NEXT: addi.d $sp, $sp, 96 +; CHECK-NEXT: ld.b $t0, $sp, 56 +; CHECK-NEXT: ld.b $t1, $sp, 48 +; CHECK-NEXT: ld.b $t2, $sp, 40 +; CHECK-NEXT: ld.b $t3, $sp, 32 +; CHECK-NEXT: ld.b $t4, $sp, 24 +; CHECK-NEXT: ld.b $t5, $sp, 16 +; CHECK-NEXT: ld.b $t6, $sp, 8 +; CHECK-NEXT: ld.b $t7, $sp, 0 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 0 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a2, 1 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a3, 2 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a4, 5 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a5, 7 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a6, 8 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a7, 15 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t7, 1 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t6, 2 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t5, 4 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t4, 6 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t3, 7 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t2, 11 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t1, 12 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t0, 15 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: %ins0 = insertelement <32 x i8> undef, i8 %a0, i32 0 @@ -427,6 +433,88 @@ entry: ret void } +define void @buildvector_v32i8_with_constant(ptr %dst, i8 %a0, i8 %a1, i8 %a2, i8 %a5, i8 %a8, i8 %a9, i8 %a15, i8 %a17, i8 %a18, i8 %a20, i8 %a22, i8 %a23, i8 %a27, i8 %a28, i8 %a31) nounwind { +; CHECK-LABEL: buildvector_v32i8_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: ld.b $t0, $sp, 56 +; CHECK-NEXT: ld.b $t1, $sp, 48 +; CHECK-NEXT: ld.b $t2, $sp, 40 +; CHECK-NEXT: ld.b $t3, $sp, 32 +; CHECK-NEXT: ld.b $t4, $sp, 24 +; CHECK-NEXT: ld.b $t5, $sp, 16 +; CHECK-NEXT: ld.b $t6, $sp, 8 +; CHECK-NEXT: ld.b $t7, $sp, 0 +; CHECK-NEXT: xvrepli.b $xr0, 0 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 0 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a2, 1 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a3, 2 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a4, 5 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a5, 8 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a6, 9 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a7, 15 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t7, 1 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t6, 2 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t5, 4 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t4, 6 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t3, 7 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t2, 11 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t1, 12 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.b $vr1, $t0, 15 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <32 x i8> undef, i8 %a0, i32 0 + %ins1 = insertelement <32 x i8> %ins0, i8 %a1, i32 1 + %ins2 = insertelement <32 x i8> %ins1, i8 %a2, i32 2 + %ins3 = insertelement <32 x i8> %ins2, i8 0, i32 3 + %ins4 = insertelement <32 x i8> %ins3, i8 0, i32 4 + %ins5 = insertelement <32 x i8> %ins4, i8 %a5, i32 5 + %ins6 = insertelement <32 x i8> %ins5, i8 undef, i32 6 + %ins7 = insertelement <32 x i8> %ins6, i8 0, i32 7 + %ins8 = insertelement <32 x i8> %ins7, i8 %a8, i32 8 + %ins9 = insertelement <32 x i8> %ins8, i8 %a9, i32 9 + %ins10 = insertelement <32 x i8> %ins9, i8 0, i32 10 + %ins11 = insertelement <32 x i8> %ins10, i8 undef, i32 11 + %ins12 = insertelement <32 x i8> %ins11, i8 0, i32 12 + %ins13 = insertelement <32 x i8> %ins12, i8 0, i32 13 + %ins14 = insertelement <32 x i8> %ins13, i8 undef, i32 14 + %ins15 = insertelement <32 x i8> %ins14, i8 %a15, i32 15 + %ins16 = insertelement <32 x i8> %ins15, i8 0, i32 16 + %ins17 = insertelement <32 x i8> %ins16, i8 %a17, i32 17 + %ins18 = insertelement <32 x i8> %ins17, i8 %a18, i32 18 + %ins19 = insertelement <32 x i8> %ins18, i8 0, i32 19 + %ins20 = insertelement <32 x i8> %ins19, i8 %a20, i32 20 + %ins21 = insertelement <32 x i8> %ins20, i8 0, i32 21 + %ins22 = insertelement <32 x i8> %ins21, i8 %a22, i32 22 + %ins23 = insertelement <32 x i8> %ins22, i8 %a23, i32 23 + %ins24 = insertelement <32 x i8> %ins23, i8 0, i32 24 + %ins25 = insertelement <32 x i8> %ins24, i8 undef, i32 25 + %ins26 = insertelement <32 x i8> %ins25, i8 undef, i32 26 + %ins27 = insertelement <32 x i8> %ins26, i8 %a27, i32 27 + %ins28 = insertelement <32 x i8> %ins27, i8 %a28, i32 28 + %ins29 = insertelement <32 x i8> %ins28, i8 0, i32 29 + %ins30 = insertelement <32 x i8> %ins29, i8 undef, i32 30 + %ins31 = insertelement <32 x i8> %ins30, i8 %a31, i32 31 + store <32 x i8> %ins31, ptr %dst + ret void +} + define void @buildvector_v16i16(ptr %dst, i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16 %a4, i16 %a5, i16 %a6, i16 %a7, i16 %a8, i16 %a9, i16 %a10, i16 %a11, i16 %a12, i16 %a13, i16 %a14, i16 %a15) nounwind { ; CHECK-LABEL: buildvector_v16i16: ; CHECK: # %bb.0: # %entry @@ -497,24 +585,18 @@ entry: define void @buildvector_v16i16_partial(ptr %dst, i16 %a0, i16 %a2, i16 %a5, i16 %a6, i16 %a7, i16 %a12, i16 %a13) nounwind { ; CHECK-LABEL: buildvector_v16i16_partial: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: addi.d $sp, $sp, -96 -; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill -; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill -; CHECK-NEXT: addi.d $fp, $sp, 96 -; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 -; CHECK-NEXT: st.h $a7, $sp, 58 -; CHECK-NEXT: st.h $a6, $sp, 56 -; CHECK-NEXT: st.h $a5, $sp, 46 -; CHECK-NEXT: st.h $a4, $sp, 44 -; CHECK-NEXT: st.h $a3, $sp, 42 -; CHECK-NEXT: st.h $a2, $sp, 36 -; CHECK-NEXT: st.h $a1, $sp, 32 -; CHECK-NEXT: xvld $xr0, $sp, 32 -; CHECK-NEXT: xvst $xr0, $a0, 0 -; CHECK-NEXT: addi.d $sp, $fp, -96 -; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload -; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload -; CHECK-NEXT: addi.d $sp, $sp, 96 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a1, 0 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a2, 2 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a3, 5 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a4, 6 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a5, 7 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a6, 4 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a7, 5 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: %ins0 = insertelement <16 x i16> undef, i16 %a0, i32 0 @@ -537,6 +619,44 @@ entry: ret void } +define void @buildvector_v16i16_with_constant(ptr %dst, i16 %a2, i16 %a3, i16 %a5, i16 %a6, i16 %a7, i16 %a12, i16 %a13) nounwind { +; CHECK-LABEL: buildvector_v16i16_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvrepli.h $xr0, 2 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a1, 2 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a2, 3 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a3, 5 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a4, 6 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a5, 7 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a6, 4 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvpermi.d $xr1, $xr0, 14 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a7, 5 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <16 x i16> undef, i16 2, i32 0 + %ins1 = insertelement <16 x i16> %ins0, i16 2, i32 1 + %ins2 = insertelement <16 x i16> %ins1, i16 %a2, i32 2 + %ins3 = insertelement <16 x i16> %ins2, i16 %a3, i32 3 + %ins4 = insertelement <16 x i16> %ins3, i16 2, i32 4 + %ins5 = insertelement <16 x i16> %ins4, i16 %a5, i32 5 + %ins6 = insertelement <16 x i16> %ins5, i16 %a6, i32 6 + %ins7 = insertelement <16 x i16> %ins6, i16 %a7, i32 7 + %ins8 = insertelement <16 x i16> %ins7, i16 2, i32 8 + %ins9 = insertelement <16 x i16> %ins8, i16 2, i32 9 + %ins10 = insertelement <16 x i16> %ins9, i16 2, i32 10 + %ins11 = insertelement <16 x i16> %ins10, i16 2, i32 11 + %ins12 = insertelement <16 x i16> %ins11, i16 %a12, i32 12 + %ins13 = insertelement <16 x i16> %ins12, i16 %a13, i32 13 + %ins14 = insertelement <16 x i16> %ins13, i16 2, i32 14 + %ins15 = insertelement <16 x i16> %ins14, i16 2, i32 15 + store <16 x i16> %ins15, ptr %dst + ret void +} + define void @buildvector_v8i32(ptr %dst, i32 %a0, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7) nounwind { ; CHECK-LABEL: buildvector_v8i32: ; CHECK: # %bb.0: # %entry @@ -567,21 +687,11 @@ entry: define void @buildvector_v8i32_partial(ptr %dst, i32 %a2, i32 %a4, i32 %a5, i32 %a6) nounwind { ; CHECK-LABEL: buildvector_v8i32_partial: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: addi.d $sp, $sp, -96 -; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill -; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill -; CHECK-NEXT: addi.d $fp, $sp, 96 -; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 -; CHECK-NEXT: st.w $a4, $sp, 56 -; CHECK-NEXT: st.w $a3, $sp, 52 -; CHECK-NEXT: st.w $a2, $sp, 48 -; CHECK-NEXT: st.w $a1, $sp, 40 -; CHECK-NEXT: xvld $xr0, $sp, 32 +; CHECK-NEXT: xvinsgr2vr.w $xr0, $a1, 2 +; CHECK-NEXT: xvinsgr2vr.w $xr0, $a2, 4 +; CHECK-NEXT: xvinsgr2vr.w $xr0, $a3, 5 +; CHECK-NEXT: xvinsgr2vr.w $xr0, $a4, 6 ; CHECK-NEXT: xvst $xr0, $a0, 0 -; CHECK-NEXT: addi.d $sp, $fp, -96 -; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload -; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload -; CHECK-NEXT: addi.d $sp, $sp, 96 ; CHECK-NEXT: ret entry: %ins0 = insertelement <8 x i32> undef, i32 undef, i32 0 @@ -596,6 +706,29 @@ entry: ret void } +define void @buildvector_v8i32_with_constant(ptr %dst, i32 %a2, i32 %a4, i32 %a5, i32 %a6) nounwind { +; CHECK-LABEL: buildvector_v8i32_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvrepli.b $xr0, 0 +; CHECK-NEXT: xvinsgr2vr.w $xr0, $a1, 2 +; CHECK-NEXT: xvinsgr2vr.w $xr0, $a2, 4 +; CHECK-NEXT: xvinsgr2vr.w $xr0, $a3, 5 +; CHECK-NEXT: xvinsgr2vr.w $xr0, $a4, 6 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <8 x i32> undef, i32 0, i32 0 + %ins1 = insertelement <8 x i32> %ins0, i32 0, i32 1 + %ins2 = insertelement <8 x i32> %ins1, i32 %a2, i32 2 + %ins3 = insertelement <8 x i32> %ins2, i32 0, i32 3 + %ins4 = insertelement <8 x i32> %ins3, i32 %a4, i32 4 + %ins5 = insertelement <8 x i32> %ins4, i32 %a5, i32 5 + %ins6 = insertelement <8 x i32> %ins5, i32 %a6, i32 6 + %ins7 = insertelement <8 x i32> %ins6, i32 0, i32 7 + store <8 x i32> %ins7, ptr %dst + ret void +} + define void @buildvector_v4i64(ptr %dst, i64 %a0, i64 %a1, i64 %a2, i64 %a3) nounwind { ; CHECK-LABEL: buildvector_v4i64: ; CHECK: # %bb.0: # %entry @@ -617,11 +750,8 @@ entry: define void @buildvector_v4i64_partial(ptr %dst, i64 %a1, i64 %a2) nounwind { ; CHECK-LABEL: buildvector_v4i64_partial: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvinsgr2vr.d $xr0, $a2, 0 -; CHECK-NEXT: xvpermi.d $xr0, $xr0, 68 -; CHECK-NEXT: xvinsgr2vr.d $xr1, $a1, 0 -; CHECK-NEXT: xvpermi.d $xr1, $xr1, 68 -; CHECK-NEXT: xvpackev.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvinsgr2vr.d $xr0, $a1, 1 +; CHECK-NEXT: xvinsgr2vr.d $xr0, $a2, 2 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -633,6 +763,23 @@ entry: ret void } +define void @buildvector_v4i64_with_constant(ptr %dst, i64 %a0, i64 %a2) nounwind { +; CHECK-LABEL: buildvector_v4i64_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvrepli.b $xr0, 0 +; CHECK-NEXT: xvinsgr2vr.d $xr0, $a1, 0 +; CHECK-NEXT: xvinsgr2vr.d $xr0, $a2, 2 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <4 x i64> undef, i64 %a0, i32 0 + %ins1 = insertelement <4 x i64> %ins0, i64 0, i32 1 + %ins2 = insertelement <4 x i64> %ins1, i64 %a2, i32 2 + %ins3 = insertelement <4 x i64> %ins2, i64 0, i32 3 + store <4 x i64> %ins3, ptr %dst + ret void +} + define void @buildvector_v8f32(ptr %dst, float %a0, float %a1, float %a2, float %a3, float %a4, float %a5, float %a6, float %a7) nounwind { ; CHECK-LABEL: buildvector_v8f32: ; CHECK: # %bb.0: # %entry @@ -669,21 +816,15 @@ entry: define void @buildvector_v8f32_partial(ptr %dst, float %a1, float %a2, float %a5, float %a7) nounwind { ; CHECK-LABEL: buildvector_v8f32_partial: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: addi.d $sp, $sp, -96 -; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill -; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill -; CHECK-NEXT: addi.d $fp, $sp, 96 -; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 -; CHECK-NEXT: fst.s $fa3, $sp, 60 -; CHECK-NEXT: fst.s $fa2, $sp, 52 -; CHECK-NEXT: fst.s $fa1, $sp, 40 -; CHECK-NEXT: fst.s $fa0, $sp, 36 -; CHECK-NEXT: xvld $xr0, $sp, 32 +; CHECK-NEXT: # kill: def $f3 killed $f3 def $xr3 +; CHECK-NEXT: # kill: def $f2 killed $f2 def $xr2 +; CHECK-NEXT: # kill: def $f1 killed $f1 def $xr1 +; CHECK-NEXT: # kill: def $f0 killed $f0 def $xr0 +; CHECK-NEXT: xvinsve0.w $xr0, $xr0, 1 +; CHECK-NEXT: xvinsve0.w $xr0, $xr1, 2 +; CHECK-NEXT: xvinsve0.w $xr0, $xr2, 5 +; CHECK-NEXT: xvinsve0.w $xr0, $xr3, 7 ; CHECK-NEXT: xvst $xr0, $a0, 0 -; CHECK-NEXT: addi.d $sp, $fp, -96 -; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload -; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload -; CHECK-NEXT: addi.d $sp, $sp, 96 ; CHECK-NEXT: ret entry: %ins0 = insertelement <8 x float> undef, float undef, i32 0 @@ -698,6 +839,34 @@ entry: ret void } +define void @buildvector_v8f32_with_constant(ptr %dst, float %a1, float %a2, float %a5, float %a7) nounwind { +; CHECK-LABEL: buildvector_v8f32_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: # kill: def $f3 killed $f3 def $xr3 +; CHECK-NEXT: # kill: def $f2 killed $f2 def $xr2 +; CHECK-NEXT: # kill: def $f1 killed $f1 def $xr1 +; CHECK-NEXT: # kill: def $f0 killed $f0 def $xr0 +; CHECK-NEXT: lu12i.w $a1, 262144 +; CHECK-NEXT: xvreplgr2vr.w $xr4, $a1 +; CHECK-NEXT: xvinsve0.w $xr4, $xr0, 1 +; CHECK-NEXT: xvinsve0.w $xr4, $xr1, 2 +; CHECK-NEXT: xvinsve0.w $xr4, $xr2, 5 +; CHECK-NEXT: xvinsve0.w $xr4, $xr3, 7 +; CHECK-NEXT: xvst $xr4, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <8 x float> undef, float 2.0, i32 0 + %ins1 = insertelement <8 x float> %ins0, float %a1, i32 1 + %ins2 = insertelement <8 x float> %ins1, float %a2, i32 2 + %ins3 = insertelement <8 x float> %ins2, float 2.0, i32 3 + %ins4 = insertelement <8 x float> %ins3, float 2.0, i32 4 + %ins5 = insertelement <8 x float> %ins4, float %a5, i32 5 + %ins6 = insertelement <8 x float> %ins5, float 2.0, i32 6 + %ins7 = insertelement <8 x float> %ins6, float %a7, i32 7 + store <8 x float> %ins7, ptr %dst + ret void +} + define void @buildvector_v4f64(ptr %dst, double %a0, double %a1, double %a2, double %a3) nounwind { ; CHECK-LABEL: buildvector_v4f64: ; CHECK: # %bb.0: # %entry @@ -724,9 +893,7 @@ define void @buildvector_v4f64_partial(ptr %dst, double %a0, double %a3) nounwin ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: # kill: def $f1_64 killed $f1_64 def $xr1 ; CHECK-NEXT: # kill: def $f0_64 killed $f0_64 def $xr0 -; CHECK-NEXT: xvpermi.d $xr0, $xr0, 68 -; CHECK-NEXT: xvpermi.d $xr1, $xr1, 68 -; CHECK-NEXT: xvpackev.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvinsve0.d $xr0, $xr1, 3 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -737,3 +904,22 @@ entry: store <4 x double> %ins3, ptr %dst ret void } + +define void @buildvector_v4f64_with_constant(ptr %dst, double %a0, double %a3) nounwind { +; CHECK-LABEL: buildvector_v4f64_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: # kill: def $f1_64 killed $f1_64 def $xr1 +; CHECK-NEXT: # kill: def $f0_64 killed $f0_64 def $xr0 +; CHECK-NEXT: xvrepli.b $xr2, 0 +; CHECK-NEXT: xvinsve0.d $xr2, $xr0, 0 +; CHECK-NEXT: xvinsve0.d $xr2, $xr1, 3 +; CHECK-NEXT: xvst $xr2, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <4 x double> undef, double %a0, i32 0 + %ins1 = insertelement <4 x double> %ins0, double 0.0, i32 1 + %ins2 = insertelement <4 x double> %ins1, double 0.0, i32 2 + %ins3 = insertelement <4 x double> %ins2, double %a3, i32 3 + store <4 x double> %ins3, ptr %dst + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll b/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll index c61b784..06d4a5d 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll @@ -524,9 +524,8 @@ define i8 @xvmsk_eq_v2i64_concat_poison(<2 x i64> %vec) { ; CHECK-NEXT: vpickve2gr.d $a0, $vr0, 0 ; CHECK-NEXT: vinsgr2vr.h $vr1, $a0, 0 ; CHECK-NEXT: vpickve2gr.d $a0, $vr0, 1 -; CHECK-NEXT: vinsgr2vr.h $vr0, $a0, 0 -; CHECK-NEXT: vpackev.h $vr0, $vr0, $vr1 -; CHECK-NEXT: vslli.h $vr0, $vr0, 15 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a0, 1 +; CHECK-NEXT: vslli.h $vr0, $vr1, 15 ; CHECK-NEXT: vmskltz.h $vr0, $vr0 ; CHECK-NEXT: vpickve2gr.hu $a0, $vr0, 0 ; CHECK-NEXT: ret @@ -539,24 +538,20 @@ define i8 @xvmsk_eq_v2i64_concat_poison(<2 x i64> %vec) { define i8 @xvmsk_ne_v4i32_concat_poison(<4 x i32> %vec) { ; CHECK-LABEL: xvmsk_ne_v4i32_concat_poison: ; CHECK: # %bb.0: -; CHECK-NEXT: addi.d $sp, $sp, -16 -; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: vseqi.w $vr0, $vr0, 0 ; CHECK-NEXT: vrepli.b $vr1, -1 ; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 -; CHECK-NEXT: vpickve2gr.w $a0, $vr0, 3 -; CHECK-NEXT: st.h $a0, $sp, 6 -; CHECK-NEXT: vpickve2gr.w $a0, $vr0, 2 -; CHECK-NEXT: st.h $a0, $sp, 4 -; CHECK-NEXT: vpickve2gr.w $a0, $vr0, 1 -; CHECK-NEXT: st.h $a0, $sp, 2 ; CHECK-NEXT: vpickve2gr.w $a0, $vr0, 0 -; CHECK-NEXT: st.h $a0, $sp, 0 -; CHECK-NEXT: vld $vr0, $sp, 0 -; CHECK-NEXT: vslli.h $vr0, $vr0, 15 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a0, 0 +; CHECK-NEXT: vpickve2gr.w $a0, $vr0, 1 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a0, 1 +; CHECK-NEXT: vpickve2gr.w $a0, $vr0, 2 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a0, 2 +; CHECK-NEXT: vpickve2gr.w $a0, $vr0, 3 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a0, 3 +; CHECK-NEXT: vslli.h $vr0, $vr1, 15 ; CHECK-NEXT: vmskltz.h $vr0, $vr0 ; CHECK-NEXT: vpickve2gr.hu $a0, $vr0, 0 -; CHECK-NEXT: addi.d $sp, $sp, 16 ; CHECK-NEXT: ret %tobool = icmp ne <4 x i32> %vec, zeroinitializer %insertvec = shufflevector <4 x i1> %tobool, <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison> @@ -567,23 +562,19 @@ define i8 @xvmsk_ne_v4i32_concat_poison(<4 x i32> %vec) { define i8 @xvmsk_ogt_v4f64_concat_poison(<4 x double> %vec) { ; CHECK-LABEL: xvmsk_ogt_v4f64_concat_poison: ; CHECK: # %bb.0: -; CHECK-NEXT: addi.d $sp, $sp, -16 -; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: xvrepli.b $xr1, 0 ; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr1, $xr0 -; CHECK-NEXT: xvpickve2gr.d $a0, $xr0, 0 -; CHECK-NEXT: xvpickve2gr.d $a1, $xr0, 1 -; CHECK-NEXT: xvpickve2gr.d $a2, $xr0, 2 -; CHECK-NEXT: xvpickve2gr.d $a3, $xr0, 3 -; CHECK-NEXT: st.h $a3, $sp, 6 -; CHECK-NEXT: st.h $a2, $sp, 4 -; CHECK-NEXT: st.h $a1, $sp, 2 -; CHECK-NEXT: st.h $a0, $sp, 0 -; CHECK-NEXT: vld $vr0, $sp, 0 +; CHECK-NEXT: xvpickve2gr.d $a0, $xr0, 3 +; CHECK-NEXT: xvpickve2gr.d $a1, $xr0, 2 +; CHECK-NEXT: xvpickve2gr.d $a2, $xr0, 1 +; CHECK-NEXT: xvpickve2gr.d $a3, $xr0, 0 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a3, 0 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a2, 1 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a1, 2 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a0, 3 ; CHECK-NEXT: vslli.h $vr0, $vr0, 15 ; CHECK-NEXT: vmskltz.h $vr0, $vr0 ; CHECK-NEXT: vpickve2gr.hu $a0, $vr0, 0 -; CHECK-NEXT: addi.d $sp, $sp, 16 ; CHECK-NEXT: ret %tobool = fcmp ogt <4 x double> %vec, zeroinitializer %insertvec = shufflevector <4 x i1> %tobool, <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison> diff --git a/llvm/test/CodeGen/LoongArch/llvm.exp10.ll b/llvm/test/CodeGen/LoongArch/llvm.exp10.ll index 62ea5cb..030b822c 100644 --- a/llvm/test/CodeGen/LoongArch/llvm.exp10.ll +++ b/llvm/test/CodeGen/LoongArch/llvm.exp10.ll @@ -137,20 +137,20 @@ define <2 x float> @exp10_v2f32(<2 x float> %x) #0 { ; LA64-NEXT: addi.d $sp, $sp, -48 ; LA64-NEXT: st.d $ra, $sp, 40 # 8-byte Folded Spill ; LA64-NEXT: vst $vr0, $sp, 0 # 16-byte Folded Spill -; LA64-NEXT: vreplvei.w $vr0, $vr0, 0 +; LA64-NEXT: vreplvei.w $vr0, $vr0, 1 ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 ; LA64-NEXT: pcaddu18i $ra, %call36(exp10f) ; LA64-NEXT: jirl $ra, $ra, 0 ; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0 ; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill ; LA64-NEXT: vld $vr0, $sp, 0 # 16-byte Folded Reload -; LA64-NEXT: vreplvei.w $vr0, $vr0, 1 +; LA64-NEXT: vreplvei.w $vr0, $vr0, 0 ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 ; LA64-NEXT: pcaddu18i $ra, %call36(exp10f) ; LA64-NEXT: jirl $ra, $ra, 0 ; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0 ; LA64-NEXT: vld $vr1, $sp, 16 # 16-byte Folded Reload -; LA64-NEXT: vpackev.w $vr0, $vr0, $vr1 +; LA64-NEXT: vextrins.w $vr0, $vr1, 16 ; LA64-NEXT: ld.d $ra, $sp, 40 # 8-byte Folded Reload ; LA64-NEXT: addi.d $sp, $sp, 48 ; LA64-NEXT: ret diff --git a/llvm/test/CodeGen/LoongArch/llvm.sincos.ll b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll index 383d63c..4ac38a9 100644 --- a/llvm/test/CodeGen/LoongArch/llvm.sincos.ll +++ b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll @@ -350,7 +350,7 @@ define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) #0 { ; LA64-NEXT: addi.d $sp, $sp, -80 ; LA64-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill ; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill -; LA64-NEXT: vreplvei.w $vr0, $vr0, 0 +; LA64-NEXT: vreplvei.w $vr0, $vr0, 1 ; LA64-NEXT: vst $vr0, $sp, 48 # 16-byte Folded Spill ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 ; LA64-NEXT: pcaddu18i $ra, %call36(sinf) @@ -358,14 +358,14 @@ define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) #0 { ; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0 ; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill ; LA64-NEXT: vld $vr0, $sp, 16 # 16-byte Folded Reload -; LA64-NEXT: vreplvei.w $vr0, $vr0, 1 +; LA64-NEXT: vreplvei.w $vr0, $vr0, 0 ; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 ; LA64-NEXT: pcaddu18i $ra, %call36(sinf) ; LA64-NEXT: jirl $ra, $ra, 0 ; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0 ; LA64-NEXT: vld $vr1, $sp, 32 # 16-byte Folded Reload -; LA64-NEXT: vpackev.w $vr0, $vr0, $vr1 +; LA64-NEXT: vextrins.w $vr0, $vr1, 16 ; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill ; LA64-NEXT: vld $vr0, $sp, 48 # 16-byte Folded Reload ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 @@ -377,9 +377,9 @@ define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) #0 { ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 ; LA64-NEXT: pcaddu18i $ra, %call36(cosf) ; LA64-NEXT: jirl $ra, $ra, 0 -; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0 -; LA64-NEXT: vld $vr1, $sp, 48 # 16-byte Folded Reload -; LA64-NEXT: vpackev.w $vr1, $vr0, $vr1 +; LA64-NEXT: fmov.s $fa1, $fa0 +; LA64-NEXT: vld $vr0, $sp, 48 # 16-byte Folded Reload +; LA64-NEXT: vextrins.w $vr1, $vr0, 16 ; LA64-NEXT: vld $vr0, $sp, 32 # 16-byte Folded Reload ; LA64-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload ; LA64-NEXT: addi.d $sp, $sp, 80 @@ -439,48 +439,60 @@ define { <3 x float>, <3 x float> } @test_sincos_v3f32(<3 x float> %a) #0 { ; ; LA64-LABEL: test_sincos_v3f32: ; LA64: # %bb.0: -; LA64-NEXT: addi.d $sp, $sp, -112 -; LA64-NEXT: st.d $ra, $sp, 104 # 8-byte Folded Spill +; LA64-NEXT: addi.d $sp, $sp, -96 +; LA64-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill ; LA64-NEXT: vst $vr0, $sp, 48 # 16-byte Folded Spill -; LA64-NEXT: vreplvei.w $vr0, $vr0, 2 +; LA64-NEXT: vreplvei.w $vr0, $vr0, 1 ; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 ; LA64-NEXT: pcaddu18i $ra, %call36(sinf) ; LA64-NEXT: jirl $ra, $ra, 0 -; LA64-NEXT: fst.s $fa0, $sp, 88 +; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0 +; LA64-NEXT: vst $vr0, $sp, 64 # 16-byte Folded Spill ; LA64-NEXT: vld $vr0, $sp, 48 # 16-byte Folded Reload -; LA64-NEXT: vreplvei.w $vr0, $vr0, 1 +; LA64-NEXT: vreplvei.w $vr0, $vr0, 0 ; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 ; LA64-NEXT: pcaddu18i $ra, %call36(sinf) ; LA64-NEXT: jirl $ra, $ra, 0 -; LA64-NEXT: fst.s $fa0, $sp, 84 +; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0 +; LA64-NEXT: vld $vr1, $sp, 64 # 16-byte Folded Reload +; LA64-NEXT: vextrins.w $vr0, $vr1, 16 +; LA64-NEXT: vst $vr0, $sp, 64 # 16-byte Folded Spill ; LA64-NEXT: vld $vr0, $sp, 48 # 16-byte Folded Reload -; LA64-NEXT: vreplvei.w $vr0, $vr0, 0 +; LA64-NEXT: vreplvei.w $vr0, $vr0, 2 ; LA64-NEXT: vst $vr0, $sp, 48 # 16-byte Folded Spill ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 ; LA64-NEXT: pcaddu18i $ra, %call36(sinf) ; LA64-NEXT: jirl $ra, $ra, 0 -; LA64-NEXT: fst.s $fa0, $sp, 80 +; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0 +; LA64-NEXT: vld $vr1, $sp, 64 # 16-byte Folded Reload +; LA64-NEXT: vextrins.w $vr1, $vr0, 32 +; LA64-NEXT: vst $vr1, $sp, 64 # 16-byte Folded Spill ; LA64-NEXT: vld $vr0, $sp, 32 # 16-byte Folded Reload ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 ; LA64-NEXT: pcaddu18i $ra, %call36(cosf) ; LA64-NEXT: jirl $ra, $ra, 0 -; LA64-NEXT: fst.s $fa0, $sp, 72 +; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0 +; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill ; LA64-NEXT: vld $vr0, $sp, 16 # 16-byte Folded Reload ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 ; LA64-NEXT: pcaddu18i $ra, %call36(cosf) ; LA64-NEXT: jirl $ra, $ra, 0 -; LA64-NEXT: fst.s $fa0, $sp, 68 +; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0 +; LA64-NEXT: vld $vr1, $sp, 32 # 16-byte Folded Reload +; LA64-NEXT: vextrins.w $vr0, $vr1, 16 +; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill ; LA64-NEXT: vld $vr0, $sp, 48 # 16-byte Folded Reload ; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0 ; LA64-NEXT: pcaddu18i $ra, %call36(cosf) ; LA64-NEXT: jirl $ra, $ra, 0 -; LA64-NEXT: fst.s $fa0, $sp, 64 -; LA64-NEXT: vld $vr0, $sp, 80 -; LA64-NEXT: vld $vr1, $sp, 64 -; LA64-NEXT: ld.d $ra, $sp, 104 # 8-byte Folded Reload -; LA64-NEXT: addi.d $sp, $sp, 112 +; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0 +; LA64-NEXT: vld $vr1, $sp, 32 # 16-byte Folded Reload +; LA64-NEXT: vextrins.w $vr1, $vr0, 32 +; LA64-NEXT: vld $vr0, $sp, 64 # 16-byte Folded Reload +; LA64-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload +; LA64-NEXT: addi.d $sp, $sp, 96 ; LA64-NEXT: ret %result = call { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float> %a) ret { <3 x float>, <3 x float> } %result diff --git a/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll b/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll index 4dda012..9517558 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll @@ -275,16 +275,13 @@ entry: define void @buildvector_v16i8_partial(ptr %dst, i8 %a2, i8 %a6, i8 %a8, i8 %a11, i8 %a12, i8 %a15) nounwind { ; CHECK-LABEL: buildvector_v16i8_partial: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: addi.d $sp, $sp, -16 -; CHECK-NEXT: st.b $a6, $sp, 15 -; CHECK-NEXT: st.b $a5, $sp, 12 -; CHECK-NEXT: st.b $a4, $sp, 11 -; CHECK-NEXT: st.b $a3, $sp, 8 -; CHECK-NEXT: st.b $a2, $sp, 6 -; CHECK-NEXT: st.b $a1, $sp, 2 -; CHECK-NEXT: vld $vr0, $sp, 0 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 2 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a2, 6 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a3, 8 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a4, 11 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a5, 12 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a6, 15 ; CHECK-NEXT: vst $vr0, $a0, 0 -; CHECK-NEXT: addi.d $sp, $sp, 16 ; CHECK-NEXT: ret entry: %ins0 = insertelement <16 x i8> undef, i8 undef, i32 0 @@ -307,6 +304,40 @@ entry: ret void } +define void @buildvector_v16i8_with_constant(ptr %dst, i8 %a0, i8 %a4, i8 %a6, i8 %a8, i8 %a11, i8 %a12, i8 %a15) nounwind { +; CHECK-LABEL: buildvector_v16i8_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vrepli.b $vr0, 0 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 0 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a2, 4 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a3, 6 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a4, 8 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a5, 11 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a6, 12 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a7, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <16 x i8> undef, i8 %a0, i32 0 + %ins1 = insertelement <16 x i8> %ins0, i8 0, i32 1 + %ins2 = insertelement <16 x i8> %ins1, i8 0, i32 2 + %ins3 = insertelement <16 x i8> %ins2, i8 0, i32 3 + %ins4 = insertelement <16 x i8> %ins3, i8 %a4, i32 4 + %ins5 = insertelement <16 x i8> %ins4, i8 0, i32 5 + %ins6 = insertelement <16 x i8> %ins5, i8 %a6, i32 6 + %ins7 = insertelement <16 x i8> %ins6, i8 0, i32 7 + %ins8 = insertelement <16 x i8> %ins7, i8 %a8, i32 8 + %ins9 = insertelement <16 x i8> %ins8, i8 0, i32 9 + %ins10 = insertelement <16 x i8> %ins9, i8 0, i32 10 + %ins11 = insertelement <16 x i8> %ins10, i8 %a11, i32 11 + %ins12 = insertelement <16 x i8> %ins11, i8 %a12, i32 12 + %ins13 = insertelement <16 x i8> %ins12, i8 0, i32 13 + %ins14 = insertelement <16 x i8> %ins13, i8 0, i32 14 + %ins15 = insertelement <16 x i8> %ins14, i8 %a15, i32 15 + store <16 x i8> %ins15, ptr %dst + ret void +} + define void @buildvector_v8i16(ptr %dst, i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16 %a4, i16 %a5, i16 %a6, i16 %a7) nounwind { ; CHECK-LABEL: buildvector_v8i16: ; CHECK: # %bb.0: # %entry @@ -337,14 +368,11 @@ entry: define void @buildvector_v8i16_partial(ptr %dst, i16 %a1, i16 %a3, i16 %a4, i16 %a5) nounwind { ; CHECK-LABEL: buildvector_v8i16_partial: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: addi.d $sp, $sp, -16 -; CHECK-NEXT: st.h $a4, $sp, 10 -; CHECK-NEXT: st.h $a3, $sp, 8 -; CHECK-NEXT: st.h $a2, $sp, 6 -; CHECK-NEXT: st.h $a1, $sp, 2 -; CHECK-NEXT: vld $vr0, $sp, 0 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a1, 1 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a2, 3 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a3, 4 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a4, 5 ; CHECK-NEXT: vst $vr0, $a0, 0 -; CHECK-NEXT: addi.d $sp, $sp, 16 ; CHECK-NEXT: ret entry: %ins0 = insertelement <8 x i16> undef, i16 undef, i32 0 @@ -359,6 +387,29 @@ entry: ret void } +define void @buildvector_v8i16_with_constant(ptr %dst, i16 %a0, i16 %a3, i16 %a4, i16 %a5) nounwind { +; CHECK-LABEL: buildvector_v8i16_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vrepli.b $vr0, 0 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a1, 0 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a2, 3 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a3, 4 +; CHECK-NEXT: vinsgr2vr.h $vr0, $a4, 5 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <8 x i16> undef, i16 %a0, i32 0 + %ins1 = insertelement <8 x i16> %ins0, i16 0, i32 1 + %ins2 = insertelement <8 x i16> %ins1, i16 undef, i32 2 + %ins3 = insertelement <8 x i16> %ins2, i16 %a3, i32 3 + %ins4 = insertelement <8 x i16> %ins3, i16 %a4, i32 4 + %ins5 = insertelement <8 x i16> %ins4, i16 %a5, i32 5 + %ins6 = insertelement <8 x i16> %ins5, i16 0, i32 6 + %ins7 = insertelement <8 x i16> %ins6, i16 undef, i32 7 + store <8 x i16> %ins7, ptr %dst + ret void +} + define void @buildvector_v4i32(ptr %dst, i32 %a0, i32 %a1, i32 %a2, i32 %a3) nounwind { ; CHECK-LABEL: buildvector_v4i32: ; CHECK: # %bb.0: # %entry @@ -380,11 +431,8 @@ entry: define void @buildvector_v4i32_partial(ptr %dst, i32 %a0, i32 %a3) nounwind { ; CHECK-LABEL: buildvector_v4i32_partial: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: pcalau12i $a3, %pc_hi20(.LCPI23_0) -; CHECK-NEXT: vld $vr0, $a3, %pc_lo12(.LCPI23_0) -; CHECK-NEXT: vinsgr2vr.w $vr1, $a1, 0 -; CHECK-NEXT: vinsgr2vr.w $vr2, $a2, 0 -; CHECK-NEXT: vshuf.w $vr0, $vr2, $vr1 +; CHECK-NEXT: vinsgr2vr.w $vr0, $a1, 0 +; CHECK-NEXT: vinsgr2vr.w $vr0, $a2, 3 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -396,6 +444,24 @@ entry: ret void } +define void @buildvector_v4i32_with_constant(ptr %dst, i32 %a0, i32 %a2, i32 %a3) nounwind { +; CHECK-LABEL: buildvector_v4i32_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vrepli.w $vr0, 2 +; CHECK-NEXT: vinsgr2vr.w $vr0, $a1, 0 +; CHECK-NEXT: vinsgr2vr.w $vr0, $a2, 2 +; CHECK-NEXT: vinsgr2vr.w $vr0, $a3, 3 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <4 x i32> undef, i32 %a0, i32 0 + %ins1 = insertelement <4 x i32> %ins0, i32 2, i32 1 + %ins2 = insertelement <4 x i32> %ins1, i32 %a2, i32 2 + %ins3 = insertelement <4 x i32> %ins2, i32 %a3, i32 3 + store <4 x i32> %ins3, ptr %dst + ret void +} + define void @buildvector_v2i64(ptr %dst, i64 %a0, i64 %a1) nounwind { ; CHECK-LABEL: buildvector_v2i64: ; CHECK: # %bb.0: # %entry @@ -423,6 +489,20 @@ entry: ret void } +define void @buildvector_v2i64_with_constant(ptr %dst, i64 %a1) nounwind { +; CHECK-LABEL: buildvector_v2i64_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vrepli.b $vr0, 0 +; CHECK-NEXT: vinsgr2vr.d $vr0, $a1, 1 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <2 x i64> undef, i64 0, i32 0 + %ins1 = insertelement <2 x i64> %ins0, i64 %a1, i32 1 + store <2 x i64> %ins1, ptr %dst + ret void +} + define void @buildvector_v4f32(ptr %dst, float %a0, float %a1, float %a2, float %a3) nounwind { ; CHECK-LABEL: buildvector_v4f32: ; CHECK: # %bb.0: # %entry @@ -447,12 +527,10 @@ entry: define void @buildvector_v4f32_partial(ptr %dst, float %a0, float %a3) nounwind { ; CHECK-LABEL: buildvector_v4f32_partial: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: pcalau12i $a1, %pc_hi20(.LCPI27_0) -; CHECK-NEXT: vld $vr2, $a1, %pc_lo12(.LCPI27_0) ; CHECK-NEXT: # kill: def $f1 killed $f1 def $vr1 ; CHECK-NEXT: # kill: def $f0 killed $f0 def $vr0 -; CHECK-NEXT: vshuf.w $vr2, $vr1, $vr0 -; CHECK-NEXT: vst $vr2, $a0, 0 +; CHECK-NEXT: vextrins.w $vr0, $vr1, 48 +; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: %ins0 = insertelement <4 x float> undef, float %a0, i32 0 @@ -463,6 +541,27 @@ entry: ret void } +define void @buildvector_v4f32_with_constant(ptr %dst, float %a1, float %a2, float %a3) nounwind { +; CHECK-LABEL: buildvector_v4f32_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: # kill: def $f2 killed $f2 def $vr2 +; CHECK-NEXT: # kill: def $f1 killed $f1 def $vr1 +; CHECK-NEXT: # kill: def $f0 killed $f0 def $vr0 +; CHECK-NEXT: vrepli.b $vr3, 0 +; CHECK-NEXT: vextrins.w $vr3, $vr0, 16 +; CHECK-NEXT: vextrins.w $vr3, $vr1, 32 +; CHECK-NEXT: vextrins.w $vr3, $vr2, 48 +; CHECK-NEXT: vst $vr3, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <4 x float> undef, float 0.0, i32 0 + %ins1 = insertelement <4 x float> %ins0, float %a1, i32 1 + %ins2 = insertelement <4 x float> %ins1, float %a2, i32 2 + %ins3 = insertelement <4 x float> %ins2, float %a3, i32 3 + store <4 x float> %ins3, ptr %dst + ret void +} + define void @buildvector_v2f64(ptr %dst, double %a0, double %a1) nounwind { ; CHECK-LABEL: buildvector_v2f64: ; CHECK: # %bb.0: # %entry @@ -482,7 +581,7 @@ define void @buildvector_v2f64_partial(ptr %dst, double %a1) nounwind { ; CHECK-LABEL: buildvector_v2f64_partial: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: # kill: def $f0_64 killed $f0_64 def $vr0 -; CHECK-NEXT: vreplvei.d $vr0, $vr0, 0 +; CHECK-NEXT: vextrins.d $vr0, $vr0, 16 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -492,6 +591,21 @@ entry: ret void } +define void @buildvector_v2f64_with_constant(ptr %dst, double %a0) nounwind { +; CHECK-LABEL: buildvector_v2f64_with_constant: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: # kill: def $f0_64 killed $f0_64 def $vr0 +; CHECK-NEXT: vldi $vr1, -1024 +; CHECK-NEXT: vpackev.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %ins0 = insertelement <2 x double> undef, double %a0, i32 0 + %ins1 = insertelement <2 x double> %ins0, double 2.0, i32 1 + store <2 x double> %ins1, ptr %dst + ret void +} + ;; If `isShuffleMaskLegal` returns true, it will lead to an infinite loop. define void @extract1_i32_zext_insert0_i64_undef(ptr %src, ptr %dst) nounwind { ; CHECK-LABEL: extract1_i32_zext_insert0_i64_undef: diff --git a/llvm/test/CodeGen/Mips/abiflags-soft-float.ll b/llvm/test/CodeGen/Mips/abiflags-soft-float.ll new file mode 100644 index 0000000..01821f2 --- /dev/null +++ b/llvm/test/CodeGen/Mips/abiflags-soft-float.ll @@ -0,0 +1,12 @@ +; RUN: llc -filetype=obj -mtriple mipsel-unknown-linux -mcpu=mips32 %s -o tmp.o +; RUN: llvm-readobj -A tmp.o | FileCheck %s -check-prefix=OBJ +; RUN: llc -filetype=asm -mtriple mipsel-unknown-linux -mcpu=mips32 %s -o - | \ +; RUN: FileCheck %s -check-prefix=ASM + +; OBJ: FP ABI: Soft float +; ASM: .module softfloat + +define dso_local void @asm_is_null() "use-soft-float"="true" { + call void asm sideeffect "", ""() + ret void +} diff --git a/llvm/test/CodeGen/Mips/llvm.frexp.ll b/llvm/test/CodeGen/Mips/llvm.frexp.ll new file mode 100644 index 0000000..3226766 --- /dev/null +++ b/llvm/test/CodeGen/Mips/llvm.frexp.ll @@ -0,0 +1,651 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=mipsel < %s | FileCheck -check-prefix=MIPSEL %s +; RUN: llc -mtriple=mips < %s | FileCheck %s -check-prefixes=SOFT-FLOAT-32 +; RUN: llc -mtriple=mips64 < %s | FileCheck %s -check-prefixes=SOFT-FLOAT-64 + +define { half, i32 } @test_frexp_f16_i32(half %a) nounwind { +; MIPSEL-LABEL: test_frexp_f16_i32: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -24 +; MIPSEL-NEXT: sw $ra, 20($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: jal __extendhfsf2 +; MIPSEL-NEXT: nop +; MIPSEL-NEXT: addiu $5, $sp, 16 +; MIPSEL-NEXT: jal frexpf +; MIPSEL-NEXT: mov.s $f12, $f0 +; MIPSEL-NEXT: jal __truncsfhf2 +; MIPSEL-NEXT: mov.s $f12, $f0 +; MIPSEL-NEXT: lw $3, 16($sp) +; MIPSEL-NEXT: lw $ra, 20($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 24 +; +; SOFT-FLOAT-32-LABEL: test_frexp_f16_i32: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -24 +; SOFT-FLOAT-32-NEXT: sw $ra, 20($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-32-NEXT: nop +; SOFT-FLOAT-32-NEXT: addiu $5, $sp, 16 +; SOFT-FLOAT-32-NEXT: jal frexpf +; SOFT-FLOAT-32-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-32-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-32-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-32-NEXT: lw $3, 16($sp) +; SOFT-FLOAT-32-NEXT: lw $ra, 20($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 24 +; +; SOFT-FLOAT-64-LABEL: test_frexp_f16_i32: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -16 +; SOFT-FLOAT-64-NEXT: sd $ra, 8($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-64-NEXT: sll $4, $4, 0 +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 4 +; SOFT-FLOAT-64-NEXT: jal frexpf +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-64-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-64-NEXT: lw $3, 4($sp) +; SOFT-FLOAT-64-NEXT: ld $ra, 8($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 16 +%result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + ret { half, i32 } %result +} + +define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) nounwind { +; MIPSEL-LABEL: test_frexp_v2f16_v2i32: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -32 +; MIPSEL-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $18, 24($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $17, 20($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 16($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: move $16, $5 +; MIPSEL-NEXT: move $17, $4 +; MIPSEL-NEXT: jal __extendhfsf2 +; MIPSEL-NEXT: srl $4, $5, 16 +; MIPSEL-NEXT: addiu $5, $17, 12 +; MIPSEL-NEXT: jal frexpf +; MIPSEL-NEXT: mov.s $f12, $f0 +; MIPSEL-NEXT: jal __truncsfhf2 +; MIPSEL-NEXT: mov.s $f12, $f0 +; MIPSEL-NEXT: move $18, $2 +; MIPSEL-NEXT: jal __extendhfsf2 +; MIPSEL-NEXT: move $4, $16 +; MIPSEL-NEXT: addiu $5, $17, 8 +; MIPSEL-NEXT: jal frexpf +; MIPSEL-NEXT: mov.s $f12, $f0 +; MIPSEL-NEXT: sh $18, 2($17) +; MIPSEL-NEXT: jal __truncsfhf2 +; MIPSEL-NEXT: mov.s $f12, $f0 +; MIPSEL-NEXT: sh $2, 0($17) +; MIPSEL-NEXT: lw $16, 16($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $17, 20($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $18, 24($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-32-LABEL: test_frexp_v2f16_v2i32: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -32 +; SOFT-FLOAT-32-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $18, 24($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $17, 20($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 16($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: move $16, $5 +; SOFT-FLOAT-32-NEXT: move $17, $4 +; SOFT-FLOAT-32-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-32-NEXT: move $4, $5 +; SOFT-FLOAT-32-NEXT: addiu $5, $17, 12 +; SOFT-FLOAT-32-NEXT: jal frexpf +; SOFT-FLOAT-32-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-32-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-32-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-32-NEXT: move $18, $2 +; SOFT-FLOAT-32-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-32-NEXT: srl $4, $16, 16 +; SOFT-FLOAT-32-NEXT: addiu $5, $17, 8 +; SOFT-FLOAT-32-NEXT: jal frexpf +; SOFT-FLOAT-32-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-32-NEXT: sh $18, 2($17) +; SOFT-FLOAT-32-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-32-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-32-NEXT: sh $2, 0($17) +; SOFT-FLOAT-32-NEXT: lw $16, 16($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $17, 20($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $18, 24($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-64-LABEL: test_frexp_v2f16_v2i32: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -32 +; SOFT-FLOAT-64-NEXT: sd $ra, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $18, 16($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $17, 8($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $16, 0($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: move $16, $5 +; SOFT-FLOAT-64-NEXT: move $17, $4 +; SOFT-FLOAT-64-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-64-NEXT: sll $4, $5, 0 +; SOFT-FLOAT-64-NEXT: daddiu $5, $17, 12 +; SOFT-FLOAT-64-NEXT: jal frexpf +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-64-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-64-NEXT: move $18, $2 +; SOFT-FLOAT-64-NEXT: sll $1, $16, 0 +; SOFT-FLOAT-64-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-64-NEXT: srl $4, $1, 16 +; SOFT-FLOAT-64-NEXT: daddiu $5, $17, 8 +; SOFT-FLOAT-64-NEXT: jal frexpf +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-64-NEXT: sh $18, 2($17) +; SOFT-FLOAT-64-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-64-NEXT: sh $2, 0($17) +; SOFT-FLOAT-64-NEXT: ld $16, 0($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $17, 8($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $18, 16($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $ra, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 32 + %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a) + ret { <2 x half>, <2 x i32> } %result +} + +define { float, i32 } @test_frexp_f32_i32(float %a) nounwind { +; MIPSEL-LABEL: test_frexp_f32_i32: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -24 +; MIPSEL-NEXT: sw $ra, 20($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: jal frexpf +; MIPSEL-NEXT: addiu $5, $sp, 16 +; MIPSEL-NEXT: lw $2, 16($sp) +; MIPSEL-NEXT: lw $ra, 20($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 24 +; +; SOFT-FLOAT-32-LABEL: test_frexp_f32_i32: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -24 +; SOFT-FLOAT-32-NEXT: sw $ra, 20($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: jal frexpf +; SOFT-FLOAT-32-NEXT: addiu $5, $sp, 16 +; SOFT-FLOAT-32-NEXT: lw $2, 16($sp) +; SOFT-FLOAT-32-NEXT: lw $ra, 20($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 24 +; +; SOFT-FLOAT-64-LABEL: test_frexp_f32_i32: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -16 +; SOFT-FLOAT-64-NEXT: sd $ra, 8($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: jal frexpf +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 4 +; SOFT-FLOAT-64-NEXT: lw $2, 4($sp) +; SOFT-FLOAT-64-NEXT: ld $ra, 8($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 16 + %result = call { float, i32 } @llvm.frexp.f32.i32(float %a) + ret { float, i32 } %result +} + +define { float, i32 } @test_frexp_f32_i32_tailcall(float %a) nounwind { +; MIPSEL-LABEL: test_frexp_f32_i32_tailcall: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -24 +; MIPSEL-NEXT: sw $ra, 20($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: jal frexpf +; MIPSEL-NEXT: addiu $5, $sp, 16 +; MIPSEL-NEXT: lw $2, 16($sp) +; MIPSEL-NEXT: lw $ra, 20($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 24 +; +; SOFT-FLOAT-32-LABEL: test_frexp_f32_i32_tailcall: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -24 +; SOFT-FLOAT-32-NEXT: sw $ra, 20($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: jal frexpf +; SOFT-FLOAT-32-NEXT: addiu $5, $sp, 16 +; SOFT-FLOAT-32-NEXT: lw $2, 16($sp) +; SOFT-FLOAT-32-NEXT: lw $ra, 20($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 24 +; +; SOFT-FLOAT-64-LABEL: test_frexp_f32_i32_tailcall: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -16 +; SOFT-FLOAT-64-NEXT: sd $ra, 8($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: jal frexpf +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 4 +; SOFT-FLOAT-64-NEXT: lw $2, 4($sp) +; SOFT-FLOAT-64-NEXT: ld $ra, 8($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 16 + %result = tail call { float, i32 } @llvm.frexp.f32.i32(float %a) + ret { float, i32 } %result +} + +define { <2 x float>, <2 x i32> } @test_frexp_v2f32_v2i32(<2 x float> %a) nounwind { +; MIPSEL-LABEL: test_frexp_v2f32_v2i32: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -32 +; MIPSEL-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $17, 24($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 20($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: move $16, $6 +; MIPSEL-NEXT: move $17, $4 +; MIPSEL-NEXT: mtc1 $7, $f12 +; MIPSEL-NEXT: jal frexpf +; MIPSEL-NEXT: addiu $5, $4, 12 +; MIPSEL-NEXT: swc1 $f0, 4($17) +; MIPSEL-NEXT: mtc1 $16, $f12 +; MIPSEL-NEXT: jal frexpf +; MIPSEL-NEXT: addiu $5, $17, 8 +; MIPSEL-NEXT: swc1 $f0, 0($17) +; MIPSEL-NEXT: lw $16, 20($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $17, 24($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-32-LABEL: test_frexp_v2f32_v2i32: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -32 +; SOFT-FLOAT-32-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $17, 24($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 20($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: move $16, $6 +; SOFT-FLOAT-32-NEXT: move $17, $4 +; SOFT-FLOAT-32-NEXT: mtc1 $7, $f12 +; SOFT-FLOAT-32-NEXT: jal frexpf +; SOFT-FLOAT-32-NEXT: addiu $5, $4, 12 +; SOFT-FLOAT-32-NEXT: swc1 $f0, 4($17) +; SOFT-FLOAT-32-NEXT: mtc1 $16, $f12 +; SOFT-FLOAT-32-NEXT: jal frexpf +; SOFT-FLOAT-32-NEXT: addiu $5, $17, 8 +; SOFT-FLOAT-32-NEXT: swc1 $f0, 0($17) +; SOFT-FLOAT-32-NEXT: lw $16, 20($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $17, 24($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-64-LABEL: test_frexp_v2f32_v2i32: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -32 +; SOFT-FLOAT-64-NEXT: sd $ra, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $16, 16($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: move $16, $4 +; SOFT-FLOAT-64-NEXT: sll $1, $16, 0 +; SOFT-FLOAT-64-NEXT: mtc1 $1, $f12 +; SOFT-FLOAT-64-NEXT: jal frexpf +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 8 +; SOFT-FLOAT-64-NEXT: mfc1 $1, $f0 +; SOFT-FLOAT-64-NEXT: dsll $1, $1, 32 +; SOFT-FLOAT-64-NEXT: dsrl $2, $16, 32 +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 12 +; SOFT-FLOAT-64-NEXT: dsrl $16, $1, 32 +; SOFT-FLOAT-64-NEXT: sll $1, $2, 0 +; SOFT-FLOAT-64-NEXT: jal frexpf +; SOFT-FLOAT-64-NEXT: mtc1 $1, $f12 +; SOFT-FLOAT-64-NEXT: mfc1 $1, $f0 +; SOFT-FLOAT-64-NEXT: dsll $1, $1, 32 +; SOFT-FLOAT-64-NEXT: or $2, $16, $1 +; SOFT-FLOAT-64-NEXT: lw $1, 12($sp) +; SOFT-FLOAT-64-NEXT: dsll $1, $1, 32 +; SOFT-FLOAT-64-NEXT: lw $3, 8($sp) +; SOFT-FLOAT-64-NEXT: dsll $3, $3, 32 +; SOFT-FLOAT-64-NEXT: dsrl $3, $3, 32 +; SOFT-FLOAT-64-NEXT: or $3, $3, $1 +; SOFT-FLOAT-64-NEXT: ld $16, 16($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $ra, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 32 + %result = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> %a) + ret { <2 x float>, <2 x i32> } %result +} + +define { double, i32 } @test_frexp_f64_i32(double %a) nounwind { +; MIPSEL-LABEL: test_frexp_f64_i32: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -24 +; MIPSEL-NEXT: sw $ra, 20($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: jal frexp +; MIPSEL-NEXT: addiu $6, $sp, 16 +; MIPSEL-NEXT: lw $2, 16($sp) +; MIPSEL-NEXT: lw $ra, 20($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 24 +; +; SOFT-FLOAT-32-LABEL: test_frexp_f64_i32: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -24 +; SOFT-FLOAT-32-NEXT: sw $ra, 20($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: jal frexp +; SOFT-FLOAT-32-NEXT: addiu $6, $sp, 16 +; SOFT-FLOAT-32-NEXT: lw $2, 16($sp) +; SOFT-FLOAT-32-NEXT: lw $ra, 20($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 24 +; +; SOFT-FLOAT-64-LABEL: test_frexp_f64_i32: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -16 +; SOFT-FLOAT-64-NEXT: sd $ra, 8($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: jal frexp +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 4 +; SOFT-FLOAT-64-NEXT: lw $2, 4($sp) +; SOFT-FLOAT-64-NEXT: ld $ra, 8($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 16 + %result = call { double, i32 } @llvm.frexp.f64.i32(double %a) + ret { double, i32 } %result +} + +define { <2 x double>, <2 x i32> } @test_frexp_v2f64_v2i32(<2 x double> %a) nounwind { +; MIPSEL-LABEL: test_frexp_v2f64_v2i32: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -48 +; MIPSEL-NEXT: sw $ra, 44($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $18, 40($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $17, 36($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 32($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: move $16, $7 +; MIPSEL-NEXT: move $17, $6 +; MIPSEL-NEXT: move $18, $4 +; MIPSEL-NEXT: lw $1, 64($sp) +; MIPSEL-NEXT: lw $2, 68($sp) +; MIPSEL-NEXT: sw $2, 28($sp) +; MIPSEL-NEXT: sw $1, 24($sp) +; MIPSEL-NEXT: addiu $6, $4, 20 +; MIPSEL-NEXT: jal frexp +; MIPSEL-NEXT: ldc1 $f12, 24($sp) +; MIPSEL-NEXT: sdc1 $f0, 8($18) +; MIPSEL-NEXT: sw $16, 20($sp) +; MIPSEL-NEXT: sw $17, 16($sp) +; MIPSEL-NEXT: addiu $6, $18, 16 +; MIPSEL-NEXT: jal frexp +; MIPSEL-NEXT: ldc1 $f12, 16($sp) +; MIPSEL-NEXT: sdc1 $f0, 0($18) +; MIPSEL-NEXT: lw $16, 32($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $17, 36($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $18, 40($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 44($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 48 +; +; SOFT-FLOAT-32-LABEL: test_frexp_v2f64_v2i32: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -48 +; SOFT-FLOAT-32-NEXT: sw $ra, 44($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $18, 40($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $17, 36($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 32($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: move $16, $7 +; SOFT-FLOAT-32-NEXT: move $17, $6 +; SOFT-FLOAT-32-NEXT: move $18, $4 +; SOFT-FLOAT-32-NEXT: lw $1, 64($sp) +; SOFT-FLOAT-32-NEXT: lw $2, 68($sp) +; SOFT-FLOAT-32-NEXT: sw $2, 28($sp) +; SOFT-FLOAT-32-NEXT: sw $1, 24($sp) +; SOFT-FLOAT-32-NEXT: addiu $6, $4, 20 +; SOFT-FLOAT-32-NEXT: jal frexp +; SOFT-FLOAT-32-NEXT: ldc1 $f12, 24($sp) +; SOFT-FLOAT-32-NEXT: sdc1 $f0, 8($18) +; SOFT-FLOAT-32-NEXT: sw $16, 20($sp) +; SOFT-FLOAT-32-NEXT: sw $17, 16($sp) +; SOFT-FLOAT-32-NEXT: addiu $6, $18, 16 +; SOFT-FLOAT-32-NEXT: jal frexp +; SOFT-FLOAT-32-NEXT: ldc1 $f12, 16($sp) +; SOFT-FLOAT-32-NEXT: sdc1 $f0, 0($18) +; SOFT-FLOAT-32-NEXT: lw $16, 32($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $17, 36($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $18, 40($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 44($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 48 +; +; SOFT-FLOAT-64-LABEL: test_frexp_v2f64_v2i32: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -32 +; SOFT-FLOAT-64-NEXT: sd $ra, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $17, 16($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $16, 8($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: move $16, $5 +; SOFT-FLOAT-64-NEXT: move $17, $4 +; SOFT-FLOAT-64-NEXT: dmtc1 $6, $f12 +; SOFT-FLOAT-64-NEXT: jal frexp +; SOFT-FLOAT-64-NEXT: daddiu $5, $4, 20 +; SOFT-FLOAT-64-NEXT: sdc1 $f0, 8($17) +; SOFT-FLOAT-64-NEXT: dmtc1 $16, $f12 +; SOFT-FLOAT-64-NEXT: jal frexp +; SOFT-FLOAT-64-NEXT: daddiu $5, $17, 16 +; SOFT-FLOAT-64-NEXT: sdc1 $f0, 0($17) +; SOFT-FLOAT-64-NEXT: ld $16, 8($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $17, 16($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $ra, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 32 + %result = call { <2 x double>, <2 x i32> } @llvm.frexp.v2f64.v2i32(<2 x double> %a) + ret { <2 x double>, <2 x i32> } %result +} + +define { fp128, i32 } @test_frexp_fp128_i32(fp128 %a) nounwind { +; MIPSEL-LABEL: test_frexp_fp128_i32: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -40 +; MIPSEL-NEXT: sw $ra, 36($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 32($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: move $1, $7 +; MIPSEL-NEXT: move $16, $4 +; MIPSEL-NEXT: addiu $2, $sp, 28 +; MIPSEL-NEXT: sw $2, 16($sp) +; MIPSEL-NEXT: lw $7, 56($sp) +; MIPSEL-NEXT: move $4, $5 +; MIPSEL-NEXT: move $5, $6 +; MIPSEL-NEXT: jal frexpl +; MIPSEL-NEXT: move $6, $1 +; MIPSEL-NEXT: sw $5, 12($16) +; MIPSEL-NEXT: sw $4, 8($16) +; MIPSEL-NEXT: sw $3, 4($16) +; MIPSEL-NEXT: sw $2, 0($16) +; MIPSEL-NEXT: lw $1, 28($sp) +; MIPSEL-NEXT: sw $1, 16($16) +; MIPSEL-NEXT: lw $16, 32($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 36($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 40 +; +; SOFT-FLOAT-32-LABEL: test_frexp_fp128_i32: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -40 +; SOFT-FLOAT-32-NEXT: sw $ra, 36($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 32($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: move $1, $7 +; SOFT-FLOAT-32-NEXT: move $16, $4 +; SOFT-FLOAT-32-NEXT: addiu $2, $sp, 28 +; SOFT-FLOAT-32-NEXT: sw $2, 16($sp) +; SOFT-FLOAT-32-NEXT: lw $7, 56($sp) +; SOFT-FLOAT-32-NEXT: move $4, $5 +; SOFT-FLOAT-32-NEXT: move $5, $6 +; SOFT-FLOAT-32-NEXT: jal frexpl +; SOFT-FLOAT-32-NEXT: move $6, $1 +; SOFT-FLOAT-32-NEXT: sw $5, 12($16) +; SOFT-FLOAT-32-NEXT: sw $4, 8($16) +; SOFT-FLOAT-32-NEXT: sw $3, 4($16) +; SOFT-FLOAT-32-NEXT: sw $2, 0($16) +; SOFT-FLOAT-32-NEXT: lw $1, 28($sp) +; SOFT-FLOAT-32-NEXT: sw $1, 16($16) +; SOFT-FLOAT-32-NEXT: lw $16, 32($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 36($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 40 +; +; SOFT-FLOAT-64-LABEL: test_frexp_fp128_i32: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -16 +; SOFT-FLOAT-64-NEXT: sd $ra, 8($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: dmfc1 $4, $f12 +; SOFT-FLOAT-64-NEXT: dmfc1 $5, $f13 +; SOFT-FLOAT-64-NEXT: jal frexpl +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 4 +; SOFT-FLOAT-64-NEXT: lw $4, 4($sp) +; SOFT-FLOAT-64-NEXT: ld $ra, 8($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 16 + %result = call { fp128, i32 } @llvm.frexp.fp128.i32(fp128 %a) + ret { fp128, i32 } %result +} + +define { <2 x fp128>, <2 x i32> } @test_frexp_v2fp128_v2i32(<2 x fp128> %a) nounwind { +; MIPSEL-LABEL: test_frexp_v2fp128_v2i32: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -48 +; MIPSEL-NEXT: sw $ra, 44($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $18, 40($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $17, 36($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 32($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: move $16, $7 +; MIPSEL-NEXT: move $17, $6 +; MIPSEL-NEXT: move $18, $4 +; MIPSEL-NEXT: addiu $1, $sp, 28 +; MIPSEL-NEXT: sw $1, 16($sp) +; MIPSEL-NEXT: lw $4, 72($sp) +; MIPSEL-NEXT: lw $5, 76($sp) +; MIPSEL-NEXT: lw $6, 80($sp) +; MIPSEL-NEXT: lw $7, 84($sp) +; MIPSEL-NEXT: jal frexpl +; MIPSEL-NEXT: nop +; MIPSEL-NEXT: addiu $1, $sp, 24 +; MIPSEL-NEXT: sw $1, 16($sp) +; MIPSEL-NEXT: lw $7, 68($sp) +; MIPSEL-NEXT: lw $6, 64($sp) +; MIPSEL-NEXT: sw $5, 28($18) +; MIPSEL-NEXT: sw $4, 24($18) +; MIPSEL-NEXT: sw $3, 20($18) +; MIPSEL-NEXT: sw $2, 16($18) +; MIPSEL-NEXT: move $4, $17 +; MIPSEL-NEXT: jal frexpl +; MIPSEL-NEXT: move $5, $16 +; MIPSEL-NEXT: sw $5, 12($18) +; MIPSEL-NEXT: sw $4, 8($18) +; MIPSEL-NEXT: sw $3, 4($18) +; MIPSEL-NEXT: sw $2, 0($18) +; MIPSEL-NEXT: lw $1, 28($sp) +; MIPSEL-NEXT: sw $1, 36($18) +; MIPSEL-NEXT: lw $1, 24($sp) +; MIPSEL-NEXT: sw $1, 32($18) +; MIPSEL-NEXT: lw $16, 32($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $17, 36($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $18, 40($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 44($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 48 +; +; SOFT-FLOAT-32-LABEL: test_frexp_v2fp128_v2i32: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -48 +; SOFT-FLOAT-32-NEXT: sw $ra, 44($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $18, 40($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $17, 36($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 32($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: move $16, $7 +; SOFT-FLOAT-32-NEXT: move $17, $6 +; SOFT-FLOAT-32-NEXT: move $18, $4 +; SOFT-FLOAT-32-NEXT: addiu $1, $sp, 28 +; SOFT-FLOAT-32-NEXT: sw $1, 16($sp) +; SOFT-FLOAT-32-NEXT: lw $4, 72($sp) +; SOFT-FLOAT-32-NEXT: lw $5, 76($sp) +; SOFT-FLOAT-32-NEXT: lw $6, 80($sp) +; SOFT-FLOAT-32-NEXT: lw $7, 84($sp) +; SOFT-FLOAT-32-NEXT: jal frexpl +; SOFT-FLOAT-32-NEXT: nop +; SOFT-FLOAT-32-NEXT: addiu $1, $sp, 24 +; SOFT-FLOAT-32-NEXT: sw $1, 16($sp) +; SOFT-FLOAT-32-NEXT: lw $7, 68($sp) +; SOFT-FLOAT-32-NEXT: lw $6, 64($sp) +; SOFT-FLOAT-32-NEXT: sw $5, 28($18) +; SOFT-FLOAT-32-NEXT: sw $4, 24($18) +; SOFT-FLOAT-32-NEXT: sw $3, 20($18) +; SOFT-FLOAT-32-NEXT: sw $2, 16($18) +; SOFT-FLOAT-32-NEXT: move $4, $17 +; SOFT-FLOAT-32-NEXT: jal frexpl +; SOFT-FLOAT-32-NEXT: move $5, $16 +; SOFT-FLOAT-32-NEXT: sw $5, 12($18) +; SOFT-FLOAT-32-NEXT: sw $4, 8($18) +; SOFT-FLOAT-32-NEXT: sw $3, 4($18) +; SOFT-FLOAT-32-NEXT: sw $2, 0($18) +; SOFT-FLOAT-32-NEXT: lw $1, 28($sp) +; SOFT-FLOAT-32-NEXT: sw $1, 36($18) +; SOFT-FLOAT-32-NEXT: lw $1, 24($sp) +; SOFT-FLOAT-32-NEXT: sw $1, 32($18) +; SOFT-FLOAT-32-NEXT: lw $16, 32($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $17, 36($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $18, 40($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 44($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 48 +; +; SOFT-FLOAT-64-LABEL: test_frexp_v2fp128_v2i32: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -64 +; SOFT-FLOAT-64-NEXT: sd $ra, 56($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $20, 48($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $19, 40($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $18, 32($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $17, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $16, 16($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: move $16, $6 +; SOFT-FLOAT-64-NEXT: move $17, $5 +; SOFT-FLOAT-64-NEXT: move $18, $4 +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 12 +; SOFT-FLOAT-64-NEXT: move $4, $7 +; SOFT-FLOAT-64-NEXT: jal frexpl +; SOFT-FLOAT-64-NEXT: move $5, $8 +; SOFT-FLOAT-64-NEXT: move $19, $2 +; SOFT-FLOAT-64-NEXT: move $20, $3 +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 8 +; SOFT-FLOAT-64-NEXT: lw $1, 12($sp) +; SOFT-FLOAT-64-NEXT: sw $1, 36($18) +; SOFT-FLOAT-64-NEXT: move $4, $17 +; SOFT-FLOAT-64-NEXT: jal frexpl +; SOFT-FLOAT-64-NEXT: move $5, $16 +; SOFT-FLOAT-64-NEXT: lw $1, 8($sp) +; SOFT-FLOAT-64-NEXT: sw $1, 32($18) +; SOFT-FLOAT-64-NEXT: sd $20, 24($18) +; SOFT-FLOAT-64-NEXT: sd $19, 16($18) +; SOFT-FLOAT-64-NEXT: sd $3, 8($18) +; SOFT-FLOAT-64-NEXT: sd $2, 0($18) +; SOFT-FLOAT-64-NEXT: ld $16, 16($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $17, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $18, 32($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $19, 40($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $20, 48($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $ra, 56($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 64 + %result = call { <2 x fp128>, <2 x i32> } @llvm.frexp.v2fp128.v2i32(<2 x fp128> %a) + ret { <2 x fp128>, <2 x i32> } %result +} + +declare { half, i32 } @llvm.frexp.f16.i32(half) #0 +declare { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half>) #0 + +declare { float, i32 } @llvm.frexp.f32.i32(float) #0 +declare { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float>) #0 + +declare { double, i32 } @llvm.frexp.f64.i32(double) #0 +declare { <2 x double>, <2 x i32> } @llvm.frexp.v2f64.v2i32(<2 x double>) #0 + +declare { fp128, i32 } @llvm.frexp.fp128.i32(fp128) #0 +declare { <2 x fp128>, <2 x i32> } @llvm.frexp.v2fp128.v2i32(<2 x fp128>) #0 + +attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } diff --git a/llvm/test/CodeGen/Mips/llvm.sincos.ll b/llvm/test/CodeGen/Mips/llvm.sincos.ll new file mode 100644 index 0000000..046be12 --- /dev/null +++ b/llvm/test/CodeGen/Mips/llvm.sincos.ll @@ -0,0 +1,1044 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=mipsel < %s | FileCheck -check-prefix=MIPSEL %s +; RUN: llc -mtriple=mips < %s | FileCheck %s -check-prefixes=SOFT-FLOAT-32 +; RUN: llc -mtriple=mips64 < %s | FileCheck %s -check-prefixes=SOFT-FLOAT-64 + +define { half, half } @test_sincos_f16(half %a) #0 { +; MIPSEL-LABEL: test_sincos_f16: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -32 +; MIPSEL-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 24($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: jal __extendhfsf2 +; MIPSEL-NEXT: nop +; MIPSEL-NEXT: addiu $5, $sp, 20 +; MIPSEL-NEXT: addiu $6, $sp, 16 +; MIPSEL-NEXT: jal sincosf +; MIPSEL-NEXT: mov.s $f12, $f0 +; MIPSEL-NEXT: jal __truncsfhf2 +; MIPSEL-NEXT: lwc1 $f12, 20($sp) +; MIPSEL-NEXT: move $16, $2 +; MIPSEL-NEXT: jal __truncsfhf2 +; MIPSEL-NEXT: lwc1 $f12, 16($sp) +; MIPSEL-NEXT: move $3, $2 +; MIPSEL-NEXT: move $2, $16 +; MIPSEL-NEXT: lw $16, 24($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-32-LABEL: test_sincos_f16: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -32 +; SOFT-FLOAT-32-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 24($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-32-NEXT: nop +; SOFT-FLOAT-32-NEXT: addiu $5, $sp, 20 +; SOFT-FLOAT-32-NEXT: addiu $6, $sp, 16 +; SOFT-FLOAT-32-NEXT: jal sincosf +; SOFT-FLOAT-32-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-32-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-32-NEXT: lwc1 $f12, 20($sp) +; SOFT-FLOAT-32-NEXT: move $16, $2 +; SOFT-FLOAT-32-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-32-NEXT: lwc1 $f12, 16($sp) +; SOFT-FLOAT-32-NEXT: move $3, $2 +; SOFT-FLOAT-32-NEXT: move $2, $16 +; SOFT-FLOAT-32-NEXT: lw $16, 24($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-64-LABEL: test_sincos_f16: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -32 +; SOFT-FLOAT-64-NEXT: sd $ra, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $16, 16($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-64-NEXT: sll $4, $4, 0 +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 12 +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 8 +; SOFT-FLOAT-64-NEXT: jal sincosf +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-64-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-64-NEXT: lwc1 $f12, 12($sp) +; SOFT-FLOAT-64-NEXT: move $16, $2 +; SOFT-FLOAT-64-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-64-NEXT: lwc1 $f12, 8($sp) +; SOFT-FLOAT-64-NEXT: move $3, $2 +; SOFT-FLOAT-64-NEXT: move $2, $16 +; SOFT-FLOAT-64-NEXT: ld $16, 16($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $ra, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 32 + %result = call { half, half } @llvm.sincos.f16(half %a) + ret { half, half } %result +} + +define half @test_sincos_f16_only_use_sin(half %a) #0 { +; MIPSEL-LABEL: test_sincos_f16_only_use_sin: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -32 +; MIPSEL-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: jal __extendhfsf2 +; MIPSEL-NEXT: nop +; MIPSEL-NEXT: addiu $5, $sp, 24 +; MIPSEL-NEXT: addiu $6, $sp, 20 +; MIPSEL-NEXT: jal sincosf +; MIPSEL-NEXT: mov.s $f12, $f0 +; MIPSEL-NEXT: jal __truncsfhf2 +; MIPSEL-NEXT: lwc1 $f12, 24($sp) +; MIPSEL-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-32-LABEL: test_sincos_f16_only_use_sin: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -32 +; SOFT-FLOAT-32-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-32-NEXT: nop +; SOFT-FLOAT-32-NEXT: addiu $5, $sp, 24 +; SOFT-FLOAT-32-NEXT: addiu $6, $sp, 20 +; SOFT-FLOAT-32-NEXT: jal sincosf +; SOFT-FLOAT-32-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-32-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-32-NEXT: lwc1 $f12, 24($sp) +; SOFT-FLOAT-32-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-64-LABEL: test_sincos_f16_only_use_sin: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -16 +; SOFT-FLOAT-64-NEXT: sd $ra, 8($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-64-NEXT: sll $4, $4, 0 +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 4 +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 0 +; SOFT-FLOAT-64-NEXT: jal sincosf +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-64-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-64-NEXT: lwc1 $f12, 4($sp) +; SOFT-FLOAT-64-NEXT: ld $ra, 8($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 16 + %result = call { half, half } @llvm.sincos.f16(half %a) + %result.0 = extractvalue { half, half } %result, 0 + ret half %result.0 +} + +define half @test_sincos_f16_only_use_cos(half %a) #0 { +; MIPSEL-LABEL: test_sincos_f16_only_use_cos: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -32 +; MIPSEL-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: jal __extendhfsf2 +; MIPSEL-NEXT: nop +; MIPSEL-NEXT: addiu $5, $sp, 24 +; MIPSEL-NEXT: addiu $6, $sp, 20 +; MIPSEL-NEXT: jal sincosf +; MIPSEL-NEXT: mov.s $f12, $f0 +; MIPSEL-NEXT: jal __truncsfhf2 +; MIPSEL-NEXT: lwc1 $f12, 20($sp) +; MIPSEL-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-32-LABEL: test_sincos_f16_only_use_cos: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -32 +; SOFT-FLOAT-32-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-32-NEXT: nop +; SOFT-FLOAT-32-NEXT: addiu $5, $sp, 24 +; SOFT-FLOAT-32-NEXT: addiu $6, $sp, 20 +; SOFT-FLOAT-32-NEXT: jal sincosf +; SOFT-FLOAT-32-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-32-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-32-NEXT: lwc1 $f12, 20($sp) +; SOFT-FLOAT-32-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-64-LABEL: test_sincos_f16_only_use_cos: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -16 +; SOFT-FLOAT-64-NEXT: sd $ra, 8($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-64-NEXT: sll $4, $4, 0 +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 4 +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 0 +; SOFT-FLOAT-64-NEXT: jal sincosf +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-64-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-64-NEXT: lwc1 $f12, 0($sp) +; SOFT-FLOAT-64-NEXT: ld $ra, 8($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 16 + %result = call { half, half } @llvm.sincos.f16(half %a) + %result.1 = extractvalue { half, half } %result, 1 + ret half %result.1 +} + +define { <2 x half>, <2 x half> } @test_sincos_v2f16(<2 x half> %a) #0 { +; MIPSEL-LABEL: test_sincos_v2f16: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -48 +; MIPSEL-NEXT: sw $ra, 44($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $17, 40($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 36($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: move $16, $5 +; MIPSEL-NEXT: move $17, $4 +; MIPSEL-NEXT: jal __extendhfsf2 +; MIPSEL-NEXT: move $4, $5 +; MIPSEL-NEXT: addiu $5, $sp, 24 +; MIPSEL-NEXT: addiu $6, $sp, 20 +; MIPSEL-NEXT: jal sincosf +; MIPSEL-NEXT: mov.s $f12, $f0 +; MIPSEL-NEXT: jal __extendhfsf2 +; MIPSEL-NEXT: srl $4, $16, 16 +; MIPSEL-NEXT: addiu $5, $sp, 32 +; MIPSEL-NEXT: addiu $6, $sp, 28 +; MIPSEL-NEXT: jal sincosf +; MIPSEL-NEXT: mov.s $f12, $f0 +; MIPSEL-NEXT: jal __truncsfhf2 +; MIPSEL-NEXT: lwc1 $f12, 20($sp) +; MIPSEL-NEXT: lwc1 $f12, 24($sp) +; MIPSEL-NEXT: jal __truncsfhf2 +; MIPSEL-NEXT: sh $2, 4($17) +; MIPSEL-NEXT: sh $2, 0($17) +; MIPSEL-NEXT: jal __truncsfhf2 +; MIPSEL-NEXT: lwc1 $f12, 28($sp) +; MIPSEL-NEXT: sh $2, 6($17) +; MIPSEL-NEXT: jal __truncsfhf2 +; MIPSEL-NEXT: lwc1 $f12, 32($sp) +; MIPSEL-NEXT: sh $2, 2($17) +; MIPSEL-NEXT: lw $16, 36($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $17, 40($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 44($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 48 +; +; SOFT-FLOAT-32-LABEL: test_sincos_v2f16: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -48 +; SOFT-FLOAT-32-NEXT: sw $ra, 44($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $17, 40($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 36($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: move $16, $5 +; SOFT-FLOAT-32-NEXT: move $17, $4 +; SOFT-FLOAT-32-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-32-NEXT: move $4, $5 +; SOFT-FLOAT-32-NEXT: addiu $5, $sp, 24 +; SOFT-FLOAT-32-NEXT: addiu $6, $sp, 20 +; SOFT-FLOAT-32-NEXT: jal sincosf +; SOFT-FLOAT-32-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-32-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-32-NEXT: srl $4, $16, 16 +; SOFT-FLOAT-32-NEXT: addiu $5, $sp, 32 +; SOFT-FLOAT-32-NEXT: addiu $6, $sp, 28 +; SOFT-FLOAT-32-NEXT: jal sincosf +; SOFT-FLOAT-32-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-32-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-32-NEXT: lwc1 $f12, 20($sp) +; SOFT-FLOAT-32-NEXT: lwc1 $f12, 24($sp) +; SOFT-FLOAT-32-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-32-NEXT: sh $2, 6($17) +; SOFT-FLOAT-32-NEXT: sh $2, 2($17) +; SOFT-FLOAT-32-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-32-NEXT: lwc1 $f12, 28($sp) +; SOFT-FLOAT-32-NEXT: sh $2, 4($17) +; SOFT-FLOAT-32-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-32-NEXT: lwc1 $f12, 32($sp) +; SOFT-FLOAT-32-NEXT: sh $2, 0($17) +; SOFT-FLOAT-32-NEXT: lw $16, 36($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $17, 40($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 44($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 48 +; +; SOFT-FLOAT-64-LABEL: test_sincos_v2f16: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -48 +; SOFT-FLOAT-64-NEXT: sd $ra, 40($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $17, 32($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $16, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: move $17, $5 +; SOFT-FLOAT-64-NEXT: move $16, $4 +; SOFT-FLOAT-64-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-64-NEXT: sll $4, $5, 0 +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 12 +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 8 +; SOFT-FLOAT-64-NEXT: jal sincosf +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-64-NEXT: sll $1, $17, 0 +; SOFT-FLOAT-64-NEXT: jal __extendhfsf2 +; SOFT-FLOAT-64-NEXT: srl $4, $1, 16 +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 20 +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 16 +; SOFT-FLOAT-64-NEXT: jal sincosf +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f0 +; SOFT-FLOAT-64-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-64-NEXT: lwc1 $f12, 8($sp) +; SOFT-FLOAT-64-NEXT: lwc1 $f12, 12($sp) +; SOFT-FLOAT-64-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-64-NEXT: sh $2, 6($16) +; SOFT-FLOAT-64-NEXT: sh $2, 2($16) +; SOFT-FLOAT-64-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-64-NEXT: lwc1 $f12, 16($sp) +; SOFT-FLOAT-64-NEXT: sh $2, 4($16) +; SOFT-FLOAT-64-NEXT: jal __truncsfhf2 +; SOFT-FLOAT-64-NEXT: lwc1 $f12, 20($sp) +; SOFT-FLOAT-64-NEXT: sh $2, 0($16) +; SOFT-FLOAT-64-NEXT: ld $16, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $17, 32($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $ra, 40($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 48 + %result = call { <2 x half>, <2 x half> } @llvm.sincos.v2f16(<2 x half> %a) + ret { <2 x half>, <2 x half> } %result +} + +define { float, float } @test_sincos_f32(float %a) #0 { +; MIPSEL-LABEL: test_sincos_f32: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -32 +; MIPSEL-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: addiu $5, $sp, 24 +; MIPSEL-NEXT: jal sincosf +; MIPSEL-NEXT: addiu $6, $sp, 20 +; MIPSEL-NEXT: lwc1 $f0, 24($sp) +; MIPSEL-NEXT: lwc1 $f2, 20($sp) +; MIPSEL-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-32-LABEL: test_sincos_f32: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -32 +; SOFT-FLOAT-32-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: addiu $5, $sp, 24 +; SOFT-FLOAT-32-NEXT: jal sincosf +; SOFT-FLOAT-32-NEXT: addiu $6, $sp, 20 +; SOFT-FLOAT-32-NEXT: lwc1 $f0, 24($sp) +; SOFT-FLOAT-32-NEXT: lwc1 $f2, 20($sp) +; SOFT-FLOAT-32-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-64-LABEL: test_sincos_f32: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -16 +; SOFT-FLOAT-64-NEXT: sd $ra, 8($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 4 +; SOFT-FLOAT-64-NEXT: jal sincosf +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 0 +; SOFT-FLOAT-64-NEXT: lwc1 $f0, 4($sp) +; SOFT-FLOAT-64-NEXT: lwc1 $f2, 0($sp) +; SOFT-FLOAT-64-NEXT: ld $ra, 8($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 16 + %result = call { float, float } @llvm.sincos.f32(float %a) + ret { float, float } %result +} + +define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) #0 { +; MIPSEL-LABEL: test_sincos_v2f32: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -32 +; MIPSEL-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $17, 24($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 20($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: move $16, $6 +; MIPSEL-NEXT: move $17, $4 +; MIPSEL-NEXT: mtc1 $7, $f12 +; MIPSEL-NEXT: addiu $5, $4, 4 +; MIPSEL-NEXT: jal sincosf +; MIPSEL-NEXT: addiu $6, $4, 12 +; MIPSEL-NEXT: mtc1 $16, $f12 +; MIPSEL-NEXT: addiu $6, $17, 8 +; MIPSEL-NEXT: jal sincosf +; MIPSEL-NEXT: move $5, $17 +; MIPSEL-NEXT: lw $16, 20($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $17, 24($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-32-LABEL: test_sincos_v2f32: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -32 +; SOFT-FLOAT-32-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $17, 24($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 20($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: move $16, $6 +; SOFT-FLOAT-32-NEXT: move $17, $4 +; SOFT-FLOAT-32-NEXT: mtc1 $7, $f12 +; SOFT-FLOAT-32-NEXT: addiu $5, $4, 4 +; SOFT-FLOAT-32-NEXT: jal sincosf +; SOFT-FLOAT-32-NEXT: addiu $6, $4, 12 +; SOFT-FLOAT-32-NEXT: mtc1 $16, $f12 +; SOFT-FLOAT-32-NEXT: addiu $6, $17, 8 +; SOFT-FLOAT-32-NEXT: jal sincosf +; SOFT-FLOAT-32-NEXT: move $5, $17 +; SOFT-FLOAT-32-NEXT: lw $16, 20($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $17, 24($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-64-LABEL: test_sincos_v2f32: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -32 +; SOFT-FLOAT-64-NEXT: sd $ra, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $16, 16($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: move $16, $4 +; SOFT-FLOAT-64-NEXT: dsrl $1, $4, 32 +; SOFT-FLOAT-64-NEXT: sll $1, $1, 0 +; SOFT-FLOAT-64-NEXT: mtc1 $1, $f12 +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 12 +; SOFT-FLOAT-64-NEXT: jal sincosf +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 8 +; SOFT-FLOAT-64-NEXT: sll $1, $16, 0 +; SOFT-FLOAT-64-NEXT: mtc1 $1, $f12 +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 4 +; SOFT-FLOAT-64-NEXT: jal sincosf +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 0 +; SOFT-FLOAT-64-NEXT: lwc1 $f0, 12($sp) +; SOFT-FLOAT-64-NEXT: mfc1 $1, $f0 +; SOFT-FLOAT-64-NEXT: dsll $1, $1, 32 +; SOFT-FLOAT-64-NEXT: lwc1 $f0, 8($sp) +; SOFT-FLOAT-64-NEXT: mfc1 $3, $f0 +; SOFT-FLOAT-64-NEXT: lwc1 $f0, 4($sp) +; SOFT-FLOAT-64-NEXT: mfc1 $2, $f0 +; SOFT-FLOAT-64-NEXT: dsll $2, $2, 32 +; SOFT-FLOAT-64-NEXT: dsrl $2, $2, 32 +; SOFT-FLOAT-64-NEXT: or $2, $2, $1 +; SOFT-FLOAT-64-NEXT: dsll $1, $3, 32 +; SOFT-FLOAT-64-NEXT: lwc1 $f0, 0($sp) +; SOFT-FLOAT-64-NEXT: mfc1 $3, $f0 +; SOFT-FLOAT-64-NEXT: dsll $3, $3, 32 +; SOFT-FLOAT-64-NEXT: dsrl $3, $3, 32 +; SOFT-FLOAT-64-NEXT: or $3, $3, $1 +; SOFT-FLOAT-64-NEXT: ld $16, 16($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $ra, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 32 +; SOFT-FLOAT-64R2-LABEL: test_sincos_v2f32: +; SOFT-FLOAT-64R2: # %bb.0: +; SOFT-FLOAT-64R2-NEXT: daddiu $sp, $sp, -32 +; SOFT-FLOAT-64R2-NEXT: sd $ra, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64R2-NEXT: sd $16, 16($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64R2-NEXT: move $16, $4 +; SOFT-FLOAT-64R2-NEXT: dsrl $1, $4, 32 +; SOFT-FLOAT-64R2-NEXT: sll $1, $1, 0 +; SOFT-FLOAT-64R2-NEXT: mtc1 $1, $f12 +; SOFT-FLOAT-64R2-NEXT: daddiu $5, $sp, 12 +; SOFT-FLOAT-64R2-NEXT: jal sincosf +; SOFT-FLOAT-64R2-NEXT: daddiu $6, $sp, 8 +; SOFT-FLOAT-64R2-NEXT: sll $1, $16, 0 +; SOFT-FLOAT-64R2-NEXT: mtc1 $1, $f12 +; SOFT-FLOAT-64R2-NEXT: daddiu $5, $sp, 4 +; SOFT-FLOAT-64R2-NEXT: jal sincosf +; SOFT-FLOAT-64R2-NEXT: daddiu $6, $sp, 0 +; SOFT-FLOAT-64R2-NEXT: lwc1 $f0, 12($sp) +; SOFT-FLOAT-64R2-NEXT: mfc1 $1, $f0 +; SOFT-FLOAT-64R2-NEXT: dsll $1, $1, 32 +; SOFT-FLOAT-64R2-NEXT: lwc1 $f0, 4($sp) +; SOFT-FLOAT-64R2-NEXT: mfc1 $2, $f0 +; SOFT-FLOAT-64R2-NEXT: dext $2, $2, 0, 32 +; SOFT-FLOAT-64R2-NEXT: lwc1 $f0, 8($sp) +; SOFT-FLOAT-64R2-NEXT: or $2, $2, $1 +; SOFT-FLOAT-64R2-NEXT: mfc1 $1, $f0 +; SOFT-FLOAT-64R2-NEXT: dsll $1, $1, 32 +; SOFT-FLOAT-64R2-NEXT: lwc1 $f0, 0($sp) +; SOFT-FLOAT-64R2-NEXT: mfc1 $3, $f0 +; SOFT-FLOAT-64R2-NEXT: dext $3, $3, 0, 32 +; SOFT-FLOAT-64R2-NEXT: or $3, $3, $1 +; SOFT-FLOAT-64R2-NEXT: ld $16, 16($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64R2-NEXT: ld $ra, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64R2-NEXT: jr $ra +; SOFT-FLOAT-64R2-NEXT: daddiu $sp, $sp, 32 + %result = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> %a) + ret { <2 x float>, <2 x float> } %result +} + +define { <3 x float>, <3 x float> } @test_sincos_v3f32(<3 x float> %a) #0 { +; MIPSEL-LABEL: test_sincos_v3f32: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -32 +; MIPSEL-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $18, 24($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $17, 20($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 16($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: move $16, $6 +; MIPSEL-NEXT: move $17, $5 +; MIPSEL-NEXT: move $18, $4 +; MIPSEL-NEXT: mtc1 $7, $f12 +; MIPSEL-NEXT: addiu $5, $4, 8 +; MIPSEL-NEXT: jal sincosf +; MIPSEL-NEXT: addiu $6, $4, 24 +; MIPSEL-NEXT: mtc1 $16, $f12 +; MIPSEL-NEXT: addiu $5, $18, 4 +; MIPSEL-NEXT: jal sincosf +; MIPSEL-NEXT: addiu $6, $18, 20 +; MIPSEL-NEXT: mtc1 $17, $f12 +; MIPSEL-NEXT: addiu $6, $18, 16 +; MIPSEL-NEXT: jal sincosf +; MIPSEL-NEXT: move $5, $18 +; MIPSEL-NEXT: lw $16, 16($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $17, 20($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $18, 24($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-32-LABEL: test_sincos_v3f32: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -32 +; SOFT-FLOAT-32-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $18, 24($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $17, 20($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 16($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: move $16, $6 +; SOFT-FLOAT-32-NEXT: move $17, $5 +; SOFT-FLOAT-32-NEXT: move $18, $4 +; SOFT-FLOAT-32-NEXT: mtc1 $7, $f12 +; SOFT-FLOAT-32-NEXT: addiu $5, $4, 8 +; SOFT-FLOAT-32-NEXT: jal sincosf +; SOFT-FLOAT-32-NEXT: addiu $6, $4, 24 +; SOFT-FLOAT-32-NEXT: mtc1 $16, $f12 +; SOFT-FLOAT-32-NEXT: addiu $5, $18, 4 +; SOFT-FLOAT-32-NEXT: jal sincosf +; SOFT-FLOAT-32-NEXT: addiu $6, $18, 20 +; SOFT-FLOAT-32-NEXT: mtc1 $17, $f12 +; SOFT-FLOAT-32-NEXT: addiu $6, $18, 16 +; SOFT-FLOAT-32-NEXT: jal sincosf +; SOFT-FLOAT-32-NEXT: move $5, $18 +; SOFT-FLOAT-32-NEXT: lw $16, 16($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $17, 20($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $18, 24($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 32 +; +; SOFT-FLOAT-64-LABEL: test_sincos_v3f32: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -48 +; SOFT-FLOAT-64-NEXT: sdc1 $f25, 40($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sdc1 $f24, 32($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $ra, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $16, 16($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: mov.s $f24, $f15 +; SOFT-FLOAT-64-NEXT: mov.s $f25, $f14 +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f13 +; SOFT-FLOAT-64-NEXT: move $16, $4 +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 4 +; SOFT-FLOAT-64-NEXT: jal sincosf +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 0 +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 12 +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 8 +; SOFT-FLOAT-64-NEXT: jal sincosf +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f25 +; SOFT-FLOAT-64-NEXT: daddiu $5, $16, 8 +; SOFT-FLOAT-64-NEXT: daddiu $6, $16, 24 +; SOFT-FLOAT-64-NEXT: jal sincosf +; SOFT-FLOAT-64-NEXT: mov.s $f12, $f24 +; SOFT-FLOAT-64-NEXT: lwc1 $f0, 0($sp) +; SOFT-FLOAT-64-NEXT: mfc1 $1, $f0 +; SOFT-FLOAT-64-NEXT: lwc1 $f0, 4($sp) +; SOFT-FLOAT-64-NEXT: mfc1 $2, $f0 +; SOFT-FLOAT-64-NEXT: dsll $1, $1, 32 +; SOFT-FLOAT-64-NEXT: lwc1 $f0, 8($sp) +; SOFT-FLOAT-64-NEXT: mfc1 $3, $f0 +; SOFT-FLOAT-64-NEXT: dsll $3, $3, 32 +; SOFT-FLOAT-64-NEXT: dsrl $3, $3, 32 +; SOFT-FLOAT-64-NEXT: or $1, $3, $1 +; SOFT-FLOAT-64-NEXT: sd $1, 16($16) +; SOFT-FLOAT-64-NEXT: dsll $1, $2, 32 +; SOFT-FLOAT-64-NEXT: lwc1 $f0, 12($sp) +; SOFT-FLOAT-64-NEXT: mfc1 $2, $f0 +; SOFT-FLOAT-64-NEXT: dsll $2, $2, 32 +; SOFT-FLOAT-64-NEXT: dsrl $2, $2, 32 +; SOFT-FLOAT-64-NEXT: or $1, $2, $1 +; SOFT-FLOAT-64-NEXT: sd $1, 0($16) +; SOFT-FLOAT-64-NEXT: ld $16, 16($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $ra, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ldc1 $f24, 32($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ldc1 $f25, 40($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 48 +; SOFT-FLOAT-64R2-LABEL: test_sincos_v3f32: +; SOFT-FLOAT-64R2: # %bb.0: +; SOFT-FLOAT-64R2-NEXT: daddiu $sp, $sp, -48 +; SOFT-FLOAT-64R2-NEXT: sdc1 $f25, 40($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64R2-NEXT: sdc1 $f24, 32($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64R2-NEXT: sd $ra, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64R2-NEXT: sd $16, 16($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64R2-NEXT: mov.s $f24, $f15 +; SOFT-FLOAT-64R2-NEXT: mov.s $f25, $f14 +; SOFT-FLOAT-64R2-NEXT: mov.s $f12, $f13 +; SOFT-FLOAT-64R2-NEXT: move $16, $4 +; SOFT-FLOAT-64R2-NEXT: daddiu $5, $sp, 4 +; SOFT-FLOAT-64R2-NEXT: jal sincosf +; SOFT-FLOAT-64R2-NEXT: daddiu $6, $sp, 0 +; SOFT-FLOAT-64R2-NEXT: daddiu $5, $sp, 12 +; SOFT-FLOAT-64R2-NEXT: daddiu $6, $sp, 8 +; SOFT-FLOAT-64R2-NEXT: jal sincosf +; SOFT-FLOAT-64R2-NEXT: mov.s $f12, $f25 +; SOFT-FLOAT-64R2-NEXT: daddiu $5, $16, 8 +; SOFT-FLOAT-64R2-NEXT: daddiu $6, $16, 24 +; SOFT-FLOAT-64R2-NEXT: jal sincosf +; SOFT-FLOAT-64R2-NEXT: mov.s $f12, $f24 +; SOFT-FLOAT-64R2-NEXT: lwc1 $f0, 0($sp) +; SOFT-FLOAT-64R2-NEXT: mfc1 $1, $f0 +; SOFT-FLOAT-64R2-NEXT: dsll $1, $1, 32 +; SOFT-FLOAT-64R2-NEXT: lwc1 $f0, 8($sp) +; SOFT-FLOAT-64R2-NEXT: mfc1 $2, $f0 +; SOFT-FLOAT-64R2-NEXT: dext $2, $2, 0, 32 +; SOFT-FLOAT-64R2-NEXT: or $1, $2, $1 +; SOFT-FLOAT-64R2-NEXT: lwc1 $f0, 4($sp) +; SOFT-FLOAT-64R2-NEXT: mfc1 $2, $f0 +; SOFT-FLOAT-64R2-NEXT: sd $1, 16($16) +; SOFT-FLOAT-64R2-NEXT: dsll $1, $2, 32 +; SOFT-FLOAT-64R2-NEXT: lwc1 $f0, 12($sp) +; SOFT-FLOAT-64R2-NEXT: mfc1 $2, $f0 +; SOFT-FLOAT-64R2-NEXT: dext $2, $2, 0, 32 +; SOFT-FLOAT-64R2-NEXT: or $1, $2, $1 +; SOFT-FLOAT-64R2-NEXT: sd $1, 0($16) +; SOFT-FLOAT-64R2-NEXT: ld $16, 16($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64R2-NEXT: ld $ra, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64R2-NEXT: ldc1 $f24, 32($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64R2-NEXT: ldc1 $f25, 40($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64R2-NEXT: jr $ra +; SOFT-FLOAT-64R2-NEXT: daddiu $sp, $sp, 48 + %result = call { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float> %a) + ret { <3 x float>, <3 x float> } %result +} + +define { double, double } @test_sincos_f64(double %a) #0 { +; MIPSEL-LABEL: test_sincos_f64: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -40 +; MIPSEL-NEXT: sw $ra, 36($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: addiu $6, $sp, 24 +; MIPSEL-NEXT: jal sincos +; MIPSEL-NEXT: addiu $7, $sp, 16 +; MIPSEL-NEXT: ldc1 $f0, 24($sp) +; MIPSEL-NEXT: ldc1 $f2, 16($sp) +; MIPSEL-NEXT: lw $ra, 36($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 40 +; +; SOFT-FLOAT-32-LABEL: test_sincos_f64: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -40 +; SOFT-FLOAT-32-NEXT: sw $ra, 36($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: addiu $6, $sp, 24 +; SOFT-FLOAT-32-NEXT: jal sincos +; SOFT-FLOAT-32-NEXT: addiu $7, $sp, 16 +; SOFT-FLOAT-32-NEXT: ldc1 $f0, 24($sp) +; SOFT-FLOAT-32-NEXT: ldc1 $f2, 16($sp) +; SOFT-FLOAT-32-NEXT: lw $ra, 36($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 40 +; +; SOFT-FLOAT-64-LABEL: test_sincos_f64: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -32 +; SOFT-FLOAT-64-NEXT: sd $ra, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: daddiu $5, $sp, 16 +; SOFT-FLOAT-64-NEXT: jal sincos +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 8 +; SOFT-FLOAT-64-NEXT: ldc1 $f0, 16($sp) +; SOFT-FLOAT-64-NEXT: ldc1 $f2, 8($sp) +; SOFT-FLOAT-64-NEXT: ld $ra, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 32 + %result = call { double, double } @llvm.sincos.f64(double %a) + ret { double, double } %result +} + +define { <2 x double>, <2 x double> } @test_sincos_v2f64(<2 x double> %a) #0 { +; MIPSEL-LABEL: test_sincos_v2f64: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -40 +; MIPSEL-NEXT: sw $ra, 36($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 32($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: move $16, $4 +; MIPSEL-NEXT: lw $1, 56($sp) +; MIPSEL-NEXT: lw $2, 60($sp) +; MIPSEL-NEXT: sw $2, 28($sp) +; MIPSEL-NEXT: sw $1, 24($sp) +; MIPSEL-NEXT: sw $7, 20($sp) +; MIPSEL-NEXT: sw $6, 16($sp) +; MIPSEL-NEXT: addiu $6, $4, 8 +; MIPSEL-NEXT: addiu $7, $4, 24 +; MIPSEL-NEXT: jal sincos +; MIPSEL-NEXT: ldc1 $f12, 24($sp) +; MIPSEL-NEXT: addiu $7, $16, 16 +; MIPSEL-NEXT: ldc1 $f12, 16($sp) +; MIPSEL-NEXT: jal sincos +; MIPSEL-NEXT: move $6, $16 +; MIPSEL-NEXT: lw $16, 32($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 36($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 40 +; +; SOFT-FLOAT-32-LABEL: test_sincos_v2f64: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -40 +; SOFT-FLOAT-32-NEXT: sw $ra, 36($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 32($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: move $16, $4 +; SOFT-FLOAT-32-NEXT: lw $1, 56($sp) +; SOFT-FLOAT-32-NEXT: lw $2, 60($sp) +; SOFT-FLOAT-32-NEXT: sw $2, 28($sp) +; SOFT-FLOAT-32-NEXT: sw $1, 24($sp) +; SOFT-FLOAT-32-NEXT: sw $7, 20($sp) +; SOFT-FLOAT-32-NEXT: sw $6, 16($sp) +; SOFT-FLOAT-32-NEXT: addiu $6, $4, 8 +; SOFT-FLOAT-32-NEXT: addiu $7, $4, 24 +; SOFT-FLOAT-32-NEXT: jal sincos +; SOFT-FLOAT-32-NEXT: ldc1 $f12, 24($sp) +; SOFT-FLOAT-32-NEXT: addiu $7, $16, 16 +; SOFT-FLOAT-32-NEXT: ldc1 $f12, 16($sp) +; SOFT-FLOAT-32-NEXT: jal sincos +; SOFT-FLOAT-32-NEXT: move $6, $16 +; SOFT-FLOAT-32-NEXT: lw $16, 32($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 36($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 40 +; +; SOFT-FLOAT-64-LABEL: test_sincos_v2f64: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -32 +; SOFT-FLOAT-64-NEXT: sd $ra, 24($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $17, 16($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $16, 8($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: move $16, $5 +; SOFT-FLOAT-64-NEXT: move $17, $4 +; SOFT-FLOAT-64-NEXT: dmtc1 $6, $f12 +; SOFT-FLOAT-64-NEXT: daddiu $5, $4, 8 +; SOFT-FLOAT-64-NEXT: jal sincos +; SOFT-FLOAT-64-NEXT: daddiu $6, $4, 24 +; SOFT-FLOAT-64-NEXT: dmtc1 $16, $f12 +; SOFT-FLOAT-64-NEXT: daddiu $6, $17, 16 +; SOFT-FLOAT-64-NEXT: jal sincos +; SOFT-FLOAT-64-NEXT: move $5, $17 +; SOFT-FLOAT-64-NEXT: ld $16, 8($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $17, 16($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $ra, 24($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 32 +; SOFT-FLOAT-32R2-LABEL: test_sincos_v2f64: +; SOFT-FLOAT-32R2: # %bb.0: +; SOFT-FLOAT-32R2-NEXT: addiu $sp, $sp, -32 +; SOFT-FLOAT-32R2-NEXT: sw $ra, 28($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32R2-NEXT: sw $18, 24($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32R2-NEXT: sw $17, 20($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32R2-NEXT: sw $16, 16($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32R2-NEXT: move $16, $7 +; SOFT-FLOAT-32R2-NEXT: move $17, $6 +; SOFT-FLOAT-32R2-NEXT: move $18, $4 +; SOFT-FLOAT-32R2-NEXT: lw $1, 48($sp) +; SOFT-FLOAT-32R2-NEXT: lw $2, 52($sp) +; SOFT-FLOAT-32R2-NEXT: mtc1 $2, $f12 +; SOFT-FLOAT-32R2-NEXT: mthc1 $1, $f12 +; SOFT-FLOAT-32R2-NEXT: addiu $6, $4, 8 +; SOFT-FLOAT-32R2-NEXT: jal sincos +; SOFT-FLOAT-32R2-NEXT: addiu $7, $4, 24 +; SOFT-FLOAT-32R2-NEXT: mtc1 $16, $f12 +; SOFT-FLOAT-32R2-NEXT: mthc1 $17, $f12 +; SOFT-FLOAT-32R2-NEXT: addiu $7, $18, 16 +; SOFT-FLOAT-32R2-NEXT: jal sincos +; SOFT-FLOAT-32R2-NEXT: move $6, $18 +; SOFT-FLOAT-32R2-NEXT: lw $16, 16($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32R2-NEXT: lw $17, 20($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32R2-NEXT: lw $18, 24($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32R2-NEXT: lw $ra, 28($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32R2-NEXT: jr $ra +; SOFT-FLOAT-32R2-NEXT: addiu $sp, $sp, 32 + %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %a) + ret { <2 x double>, <2 x double> } %result +} + +define { fp128, fp128 } @test_sincos_f128(fp128 %a) #0 { +; MIPSEL-LABEL: test_sincos_f128: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -64 +; MIPSEL-NEXT: sw $ra, 60($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 56($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: move $1, $7 +; MIPSEL-NEXT: move $16, $4 +; MIPSEL-NEXT: addiu $2, $sp, 24 +; MIPSEL-NEXT: sw $2, 20($sp) +; MIPSEL-NEXT: addiu $2, $sp, 40 +; MIPSEL-NEXT: sw $2, 16($sp) +; MIPSEL-NEXT: lw $7, 80($sp) +; MIPSEL-NEXT: move $4, $5 +; MIPSEL-NEXT: move $5, $6 +; MIPSEL-NEXT: jal sincosl +; MIPSEL-NEXT: move $6, $1 +; MIPSEL-NEXT: lw $1, 52($sp) +; MIPSEL-NEXT: lw $2, 24($sp) +; MIPSEL-NEXT: lw $3, 28($sp) +; MIPSEL-NEXT: lw $4, 32($sp) +; MIPSEL-NEXT: lw $5, 36($sp) +; MIPSEL-NEXT: sw $5, 28($16) +; MIPSEL-NEXT: sw $4, 24($16) +; MIPSEL-NEXT: sw $3, 20($16) +; MIPSEL-NEXT: sw $2, 16($16) +; MIPSEL-NEXT: sw $1, 12($16) +; MIPSEL-NEXT: lw $1, 48($sp) +; MIPSEL-NEXT: sw $1, 8($16) +; MIPSEL-NEXT: lw $1, 44($sp) +; MIPSEL-NEXT: sw $1, 4($16) +; MIPSEL-NEXT: lw $1, 40($sp) +; MIPSEL-NEXT: sw $1, 0($16) +; MIPSEL-NEXT: lw $16, 56($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 60($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 64 +; +; SOFT-FLOAT-32-LABEL: test_sincos_f128: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -64 +; SOFT-FLOAT-32-NEXT: sw $ra, 60($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 56($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: move $1, $7 +; SOFT-FLOAT-32-NEXT: move $16, $4 +; SOFT-FLOAT-32-NEXT: addiu $2, $sp, 24 +; SOFT-FLOAT-32-NEXT: sw $2, 20($sp) +; SOFT-FLOAT-32-NEXT: addiu $2, $sp, 40 +; SOFT-FLOAT-32-NEXT: sw $2, 16($sp) +; SOFT-FLOAT-32-NEXT: lw $7, 80($sp) +; SOFT-FLOAT-32-NEXT: move $4, $5 +; SOFT-FLOAT-32-NEXT: move $5, $6 +; SOFT-FLOAT-32-NEXT: jal sincosl +; SOFT-FLOAT-32-NEXT: move $6, $1 +; SOFT-FLOAT-32-NEXT: lw $1, 52($sp) +; SOFT-FLOAT-32-NEXT: lw $2, 24($sp) +; SOFT-FLOAT-32-NEXT: lw $3, 28($sp) +; SOFT-FLOAT-32-NEXT: lw $4, 32($sp) +; SOFT-FLOAT-32-NEXT: lw $5, 36($sp) +; SOFT-FLOAT-32-NEXT: sw $5, 28($16) +; SOFT-FLOAT-32-NEXT: sw $4, 24($16) +; SOFT-FLOAT-32-NEXT: sw $3, 20($16) +; SOFT-FLOAT-32-NEXT: sw $2, 16($16) +; SOFT-FLOAT-32-NEXT: sw $1, 12($16) +; SOFT-FLOAT-32-NEXT: lw $1, 48($sp) +; SOFT-FLOAT-32-NEXT: sw $1, 8($16) +; SOFT-FLOAT-32-NEXT: lw $1, 44($sp) +; SOFT-FLOAT-32-NEXT: sw $1, 4($16) +; SOFT-FLOAT-32-NEXT: lw $1, 40($sp) +; SOFT-FLOAT-32-NEXT: sw $1, 0($16) +; SOFT-FLOAT-32-NEXT: lw $16, 56($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 60($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 64 +; +; SOFT-FLOAT-64-LABEL: test_sincos_f128: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -48 +; SOFT-FLOAT-64-NEXT: sd $ra, 40($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $16, 32($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: move $16, $4 +; SOFT-FLOAT-64-NEXT: dmfc1 $4, $f13 +; SOFT-FLOAT-64-NEXT: dmfc1 $5, $f14 +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 16 +; SOFT-FLOAT-64-NEXT: jal sincosl +; SOFT-FLOAT-64-NEXT: daddiu $7, $sp, 0 +; SOFT-FLOAT-64-NEXT: ld $1, 8($sp) +; SOFT-FLOAT-64-NEXT: sd $1, 24($16) +; SOFT-FLOAT-64-NEXT: ld $1, 0($sp) +; SOFT-FLOAT-64-NEXT: sd $1, 16($16) +; SOFT-FLOAT-64-NEXT: ld $1, 24($sp) +; SOFT-FLOAT-64-NEXT: sd $1, 8($16) +; SOFT-FLOAT-64-NEXT: ld $1, 16($sp) +; SOFT-FLOAT-64-NEXT: sd $1, 0($16) +; SOFT-FLOAT-64-NEXT: ld $16, 32($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $ra, 40($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 48 + %result = call { fp128, fp128 } @llvm.sincos.f128(fp128 %a) + ret { fp128, fp128 } %result +} + +define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 { +; MIPSEL-LABEL: test_sincos_v2f128: +; MIPSEL: # %bb.0: +; MIPSEL-NEXT: addiu $sp, $sp, -96 +; MIPSEL-NEXT: sw $ra, 92($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: sw $16, 88($sp) # 4-byte Folded Spill +; MIPSEL-NEXT: move $5, $7 +; MIPSEL-NEXT: move $1, $6 +; MIPSEL-NEXT: move $16, $4 +; MIPSEL-NEXT: addiu $2, $sp, 24 +; MIPSEL-NEXT: sw $2, 20($sp) +; MIPSEL-NEXT: addiu $2, $sp, 40 +; MIPSEL-NEXT: sw $2, 16($sp) +; MIPSEL-NEXT: lw $6, 112($sp) +; MIPSEL-NEXT: lw $7, 116($sp) +; MIPSEL-NEXT: jal sincosl +; MIPSEL-NEXT: move $4, $1 +; MIPSEL-NEXT: addiu $1, $sp, 56 +; MIPSEL-NEXT: sw $1, 20($sp) +; MIPSEL-NEXT: addiu $1, $sp, 72 +; MIPSEL-NEXT: sw $1, 16($sp) +; MIPSEL-NEXT: lw $4, 120($sp) +; MIPSEL-NEXT: lw $5, 124($sp) +; MIPSEL-NEXT: lw $6, 128($sp) +; MIPSEL-NEXT: lw $7, 132($sp) +; MIPSEL-NEXT: jal sincosl +; MIPSEL-NEXT: nop +; MIPSEL-NEXT: lw $1, 36($sp) +; MIPSEL-NEXT: lw $2, 56($sp) +; MIPSEL-NEXT: lw $3, 60($sp) +; MIPSEL-NEXT: lw $4, 64($sp) +; MIPSEL-NEXT: lw $5, 52($sp) +; MIPSEL-NEXT: lw $6, 72($sp) +; MIPSEL-NEXT: lw $7, 76($sp) +; MIPSEL-NEXT: lw $8, 80($sp) +; MIPSEL-NEXT: lw $9, 84($sp) +; MIPSEL-NEXT: lw $10, 24($sp) +; MIPSEL-NEXT: lw $11, 28($sp) +; MIPSEL-NEXT: lw $12, 32($sp) +; MIPSEL-NEXT: lw $13, 68($sp) +; MIPSEL-NEXT: sw $13, 60($16) +; MIPSEL-NEXT: sw $4, 56($16) +; MIPSEL-NEXT: sw $3, 52($16) +; MIPSEL-NEXT: sw $2, 48($16) +; MIPSEL-NEXT: sw $1, 44($16) +; MIPSEL-NEXT: sw $12, 40($16) +; MIPSEL-NEXT: sw $11, 36($16) +; MIPSEL-NEXT: sw $10, 32($16) +; MIPSEL-NEXT: sw $9, 28($16) +; MIPSEL-NEXT: sw $8, 24($16) +; MIPSEL-NEXT: sw $7, 20($16) +; MIPSEL-NEXT: sw $6, 16($16) +; MIPSEL-NEXT: sw $5, 12($16) +; MIPSEL-NEXT: lw $1, 48($sp) +; MIPSEL-NEXT: sw $1, 8($16) +; MIPSEL-NEXT: lw $1, 44($sp) +; MIPSEL-NEXT: sw $1, 4($16) +; MIPSEL-NEXT: lw $1, 40($sp) +; MIPSEL-NEXT: sw $1, 0($16) +; MIPSEL-NEXT: lw $16, 88($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: lw $ra, 92($sp) # 4-byte Folded Reload +; MIPSEL-NEXT: jr $ra +; MIPSEL-NEXT: addiu $sp, $sp, 96 +; +; SOFT-FLOAT-32-LABEL: test_sincos_v2f128: +; SOFT-FLOAT-32: # %bb.0: +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, -96 +; SOFT-FLOAT-32-NEXT: sw $ra, 92($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: sw $16, 88($sp) # 4-byte Folded Spill +; SOFT-FLOAT-32-NEXT: move $5, $7 +; SOFT-FLOAT-32-NEXT: move $1, $6 +; SOFT-FLOAT-32-NEXT: move $16, $4 +; SOFT-FLOAT-32-NEXT: addiu $2, $sp, 24 +; SOFT-FLOAT-32-NEXT: sw $2, 20($sp) +; SOFT-FLOAT-32-NEXT: addiu $2, $sp, 40 +; SOFT-FLOAT-32-NEXT: sw $2, 16($sp) +; SOFT-FLOAT-32-NEXT: lw $6, 112($sp) +; SOFT-FLOAT-32-NEXT: lw $7, 116($sp) +; SOFT-FLOAT-32-NEXT: jal sincosl +; SOFT-FLOAT-32-NEXT: move $4, $1 +; SOFT-FLOAT-32-NEXT: addiu $1, $sp, 56 +; SOFT-FLOAT-32-NEXT: sw $1, 20($sp) +; SOFT-FLOAT-32-NEXT: addiu $1, $sp, 72 +; SOFT-FLOAT-32-NEXT: sw $1, 16($sp) +; SOFT-FLOAT-32-NEXT: lw $4, 120($sp) +; SOFT-FLOAT-32-NEXT: lw $5, 124($sp) +; SOFT-FLOAT-32-NEXT: lw $6, 128($sp) +; SOFT-FLOAT-32-NEXT: lw $7, 132($sp) +; SOFT-FLOAT-32-NEXT: jal sincosl +; SOFT-FLOAT-32-NEXT: nop +; SOFT-FLOAT-32-NEXT: lw $1, 36($sp) +; SOFT-FLOAT-32-NEXT: lw $2, 56($sp) +; SOFT-FLOAT-32-NEXT: lw $3, 60($sp) +; SOFT-FLOAT-32-NEXT: lw $4, 64($sp) +; SOFT-FLOAT-32-NEXT: lw $5, 52($sp) +; SOFT-FLOAT-32-NEXT: lw $6, 72($sp) +; SOFT-FLOAT-32-NEXT: lw $7, 76($sp) +; SOFT-FLOAT-32-NEXT: lw $8, 80($sp) +; SOFT-FLOAT-32-NEXT: lw $9, 84($sp) +; SOFT-FLOAT-32-NEXT: lw $10, 24($sp) +; SOFT-FLOAT-32-NEXT: lw $11, 28($sp) +; SOFT-FLOAT-32-NEXT: lw $12, 32($sp) +; SOFT-FLOAT-32-NEXT: lw $13, 68($sp) +; SOFT-FLOAT-32-NEXT: sw $13, 60($16) +; SOFT-FLOAT-32-NEXT: sw $4, 56($16) +; SOFT-FLOAT-32-NEXT: sw $3, 52($16) +; SOFT-FLOAT-32-NEXT: sw $2, 48($16) +; SOFT-FLOAT-32-NEXT: sw $1, 44($16) +; SOFT-FLOAT-32-NEXT: sw $12, 40($16) +; SOFT-FLOAT-32-NEXT: sw $11, 36($16) +; SOFT-FLOAT-32-NEXT: sw $10, 32($16) +; SOFT-FLOAT-32-NEXT: sw $9, 28($16) +; SOFT-FLOAT-32-NEXT: sw $8, 24($16) +; SOFT-FLOAT-32-NEXT: sw $7, 20($16) +; SOFT-FLOAT-32-NEXT: sw $6, 16($16) +; SOFT-FLOAT-32-NEXT: sw $5, 12($16) +; SOFT-FLOAT-32-NEXT: lw $1, 48($sp) +; SOFT-FLOAT-32-NEXT: sw $1, 8($16) +; SOFT-FLOAT-32-NEXT: lw $1, 44($sp) +; SOFT-FLOAT-32-NEXT: sw $1, 4($16) +; SOFT-FLOAT-32-NEXT: lw $1, 40($sp) +; SOFT-FLOAT-32-NEXT: sw $1, 0($16) +; SOFT-FLOAT-32-NEXT: lw $16, 88($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: lw $ra, 92($sp) # 4-byte Folded Reload +; SOFT-FLOAT-32-NEXT: jr $ra +; SOFT-FLOAT-32-NEXT: addiu $sp, $sp, 96 +; +; SOFT-FLOAT-64-LABEL: test_sincos_v2f128: +; SOFT-FLOAT-64: # %bb.0: +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, -96 +; SOFT-FLOAT-64-NEXT: sd $ra, 88($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $18, 80($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $17, 72($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: sd $16, 64($sp) # 8-byte Folded Spill +; SOFT-FLOAT-64-NEXT: move $1, $7 +; SOFT-FLOAT-64-NEXT: move $16, $6 +; SOFT-FLOAT-64-NEXT: move $17, $5 +; SOFT-FLOAT-64-NEXT: move $18, $4 +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 48 +; SOFT-FLOAT-64-NEXT: daddiu $7, $sp, 32 +; SOFT-FLOAT-64-NEXT: move $4, $1 +; SOFT-FLOAT-64-NEXT: jal sincosl +; SOFT-FLOAT-64-NEXT: move $5, $8 +; SOFT-FLOAT-64-NEXT: daddiu $6, $sp, 16 +; SOFT-FLOAT-64-NEXT: daddiu $7, $sp, 0 +; SOFT-FLOAT-64-NEXT: move $4, $17 +; SOFT-FLOAT-64-NEXT: jal sincosl +; SOFT-FLOAT-64-NEXT: move $5, $16 +; SOFT-FLOAT-64-NEXT: ld $1, 56($sp) +; SOFT-FLOAT-64-NEXT: ld $2, 0($sp) +; SOFT-FLOAT-64-NEXT: ld $3, 8($sp) +; SOFT-FLOAT-64-NEXT: ld $4, 32($sp) +; SOFT-FLOAT-64-NEXT: ld $5, 40($sp) +; SOFT-FLOAT-64-NEXT: sd $5, 56($18) +; SOFT-FLOAT-64-NEXT: sd $4, 48($18) +; SOFT-FLOAT-64-NEXT: sd $3, 40($18) +; SOFT-FLOAT-64-NEXT: sd $2, 32($18) +; SOFT-FLOAT-64-NEXT: sd $1, 24($18) +; SOFT-FLOAT-64-NEXT: ld $1, 48($sp) +; SOFT-FLOAT-64-NEXT: sd $1, 16($18) +; SOFT-FLOAT-64-NEXT: ld $1, 24($sp) +; SOFT-FLOAT-64-NEXT: sd $1, 8($18) +; SOFT-FLOAT-64-NEXT: ld $1, 16($sp) +; SOFT-FLOAT-64-NEXT: sd $1, 0($18) +; SOFT-FLOAT-64-NEXT: ld $16, 64($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $17, 72($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $18, 80($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: ld $ra, 88($sp) # 8-byte Folded Reload +; SOFT-FLOAT-64-NEXT: jr $ra +; SOFT-FLOAT-64-NEXT: daddiu $sp, $sp, 96 + %result = call { <2 x fp128>, <2 x fp128> } @llvm.sincos.v2f128(<2 x fp128> %a) + ret { <2 x fp128>, <2 x fp128> } %result +} + +attributes #0 = { nounwind } diff --git a/llvm/test/CodeGen/Mips/nan_lowering.ll b/llvm/test/CodeGen/Mips/nan_lowering.ll new file mode 100644 index 0000000..2a11278 --- /dev/null +++ b/llvm/test/CodeGen/Mips/nan_lowering.ll @@ -0,0 +1,25 @@ +; RUN: llc -mtriple=mips-linux-gnu -mattr=-nan2008 < %s | FileCheck %s +; RUN: llc -mtriple=mips-linux-gnu -mattr=+nan2008 < %s | FileCheck %s + +; Make sure that lowering does not corrupt the value of NaN values, +; regardless of what the NaN mode is. + +define float @test1() { +; CHECK: .4byte 0x7fc00000 + ret float bitcast (i32 u0x7fc00000 to float) +} + +define float @test2() { +; CHECK: .4byte 0x7fc00001 + ret float bitcast (i32 u0x7fc00001 to float) +} + +define float @test3() { +; CHECK: .4byte 0x7f800000 + ret float bitcast (i32 u0x7f800000 to float) +} + +define float @test4() { +; CHECK: .4byte 0x7f800001 + ret float bitcast (i32 u0x7f800001 to float) +} diff --git a/llvm/test/CodeGen/Mips/qnan.ll b/llvm/test/CodeGen/Mips/qnan.ll deleted file mode 100644 index e5b4aa1..0000000 --- a/llvm/test/CodeGen/Mips/qnan.ll +++ /dev/null @@ -1,14 +0,0 @@ -; RUN: llc -O3 -mcpu=mips32r2 -mtriple=mips-linux-gnu < %s -o - | FileCheck %s -check-prefixes=MIPS_Legacy -; RUN: llc -O3 -mcpu=mips32r2 -mtriple=mips-linux-gnu -mattr=+nan2008 < %s -o - | FileCheck %s -check-prefixes=MIPS_NaN2008 - -define dso_local float @nan(float noundef %a, float noundef %b) local_unnamed_addr #0 { -; MIPS_Legacy: $CPI0_0: -; MIPS_Legacy-NEXT: .4byte 0x7fa00000 # float NaN - -; MIPS_NaN2008: $CPI0_0: -; MIPS_NaN2008-NEXT: .4byte 0x7fc00000 # float NaN - -entry: - %0 = tail call float @llvm.minimum.f32(float %a, float %b) - ret float %0 -} diff --git a/llvm/test/CodeGen/PowerPC/froundeven-legalization.ll b/llvm/test/CodeGen/PowerPC/froundeven-legalization.ll new file mode 100644 index 0000000..238e200 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/froundeven-legalization.ll @@ -0,0 +1,111 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=powerpc64le < %s | FileCheck %s + +define void @test(ptr %p1, ptr %p2) nounwind { +; CHECK-LABEL: test: +; CHECK: # %bb.0: +; CHECK-NEXT: mflr 0 +; CHECK-NEXT: stdu 1, -224(1) +; CHECK-NEXT: li 5, 48 +; CHECK-NEXT: std 0, 240(1) +; CHECK-NEXT: std 27, 184(1) # 8-byte Folded Spill +; CHECK-NEXT: li 27, 16 +; CHECK-NEXT: std 28, 192(1) # 8-byte Folded Spill +; CHECK-NEXT: std 29, 200(1) # 8-byte Folded Spill +; CHECK-NEXT: li 29, 32 +; CHECK-NEXT: li 28, 48 +; CHECK-NEXT: stxvd2x 56, 1, 5 # 16-byte Folded Spill +; CHECK-NEXT: li 5, 64 +; CHECK-NEXT: std 30, 208(1) # 8-byte Folded Spill +; CHECK-NEXT: mr 30, 4 +; CHECK-NEXT: stxvd2x 57, 1, 5 # 16-byte Folded Spill +; CHECK-NEXT: li 5, 80 +; CHECK-NEXT: stxvd2x 58, 1, 5 # 16-byte Folded Spill +; CHECK-NEXT: li 5, 96 +; CHECK-NEXT: lxvd2x 58, 0, 3 +; CHECK-NEXT: stxvd2x 59, 1, 5 # 16-byte Folded Spill +; CHECK-NEXT: li 5, 112 +; CHECK-NEXT: lxvd2x 59, 3, 27 +; CHECK-NEXT: stxvd2x 60, 1, 5 # 16-byte Folded Spill +; CHECK-NEXT: li 5, 128 +; CHECK-NEXT: stxvd2x 61, 1, 5 # 16-byte Folded Spill +; CHECK-NEXT: li 5, 144 +; CHECK-NEXT: stxvd2x 62, 1, 5 # 16-byte Folded Spill +; CHECK-NEXT: li 5, 160 +; CHECK-NEXT: lxvd2x 62, 3, 28 +; CHECK-NEXT: stxvd2x 63, 1, 5 # 16-byte Folded Spill +; CHECK-NEXT: lxvd2x 63, 3, 29 +; CHECK-NEXT: xxswapd 57, 58 +; CHECK-NEXT: xxswapd 1, 59 +; CHECK-NEXT: xxswapd 60, 62 +; CHECK-NEXT: xxswapd 61, 63 +; CHECK-NEXT: bl roundeven +; CHECK-NEXT: nop +; CHECK-NEXT: xxswapd 56, 1 +; CHECK-NEXT: xxlor 1, 59, 59 +; CHECK-NEXT: bl roundeven +; CHECK-NEXT: nop +; CHECK-NEXT: xxswapd 0, 1 +; CHECK-NEXT: xxlor 1, 60, 60 +; CHECK-NEXT: xxmrgld 59, 0, 56 +; CHECK-NEXT: bl roundeven +; CHECK-NEXT: nop +; CHECK-NEXT: xxswapd 60, 1 +; CHECK-NEXT: xxlor 1, 62, 62 +; CHECK-NEXT: bl roundeven +; CHECK-NEXT: nop +; CHECK-NEXT: xxswapd 0, 1 +; CHECK-NEXT: xxlor 1, 61, 61 +; CHECK-NEXT: xxmrgld 62, 0, 60 +; CHECK-NEXT: bl roundeven +; CHECK-NEXT: nop +; CHECK-NEXT: xxswapd 61, 1 +; CHECK-NEXT: xxlor 1, 63, 63 +; CHECK-NEXT: bl roundeven +; CHECK-NEXT: nop +; CHECK-NEXT: xxswapd 0, 1 +; CHECK-NEXT: xxlor 1, 57, 57 +; CHECK-NEXT: xxmrgld 63, 0, 61 +; CHECK-NEXT: bl roundeven +; CHECK-NEXT: nop +; CHECK-NEXT: xxswapd 61, 1 +; CHECK-NEXT: xxlor 1, 58, 58 +; CHECK-NEXT: bl roundeven +; CHECK-NEXT: nop +; CHECK-NEXT: li 3, 160 +; CHECK-NEXT: stxvd2x 63, 30, 29 +; CHECK-NEXT: xxswapd 0, 1 +; CHECK-NEXT: stxvd2x 62, 30, 28 +; CHECK-NEXT: stxvd2x 59, 30, 27 +; CHECK-NEXT: ld 29, 200(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 28, 192(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 27, 184(1) # 8-byte Folded Reload +; CHECK-NEXT: lxvd2x 63, 1, 3 # 16-byte Folded Reload +; CHECK-NEXT: li 3, 144 +; CHECK-NEXT: xxmrgld 0, 0, 61 +; CHECK-NEXT: lxvd2x 62, 1, 3 # 16-byte Folded Reload +; CHECK-NEXT: li 3, 128 +; CHECK-NEXT: stxvd2x 0, 0, 30 +; CHECK-NEXT: ld 30, 208(1) # 8-byte Folded Reload +; CHECK-NEXT: lxvd2x 61, 1, 3 # 16-byte Folded Reload +; CHECK-NEXT: li 3, 112 +; CHECK-NEXT: lxvd2x 60, 1, 3 # 16-byte Folded Reload +; CHECK-NEXT: li 3, 96 +; CHECK-NEXT: lxvd2x 59, 1, 3 # 16-byte Folded Reload +; CHECK-NEXT: li 3, 80 +; CHECK-NEXT: lxvd2x 58, 1, 3 # 16-byte Folded Reload +; CHECK-NEXT: li 3, 64 +; CHECK-NEXT: lxvd2x 57, 1, 3 # 16-byte Folded Reload +; CHECK-NEXT: li 3, 48 +; CHECK-NEXT: lxvd2x 56, 1, 3 # 16-byte Folded Reload +; CHECK-NEXT: addi 1, 1, 224 +; CHECK-NEXT: ld 0, 16(1) +; CHECK-NEXT: mtlr 0 +; CHECK-NEXT: blr + %v = load <8 x double>, ptr %p1, align 64 + %res = call <8 x double> @llvm.roundeven.v8f64(<8 x double> %v) + store <8 x double> %res, ptr %p2, align 64 + ret void +} + +declare <8 x double> @llvm.roundeven.v8f64(<8 x double>) diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll index 807651c..dc80225 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll @@ -261,7 +261,7 @@ define { <8 x i8>, <8 x i8>, <8 x i8> } @vector_deinterleave_load_factor3(ptr %p ; CHECK-LABEL: vector_deinterleave_load_factor3: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vlseg3e8.v v6, (a0) +; CHECK-NEXT: vlseg3e8.v v8, (a0) ; CHECK-NEXT: ret %vec = load <24 x i8>, ptr %p %d0 = call {<8 x i8>, <8 x i8>, <8 x i8>} @llvm.vector.deinterleave3(<24 x i8> %vec) @@ -269,8 +269,8 @@ define { <8 x i8>, <8 x i8>, <8 x i8> } @vector_deinterleave_load_factor3(ptr %p %t1 = extractvalue {<8 x i8>, <8 x i8>, <8 x i8>} %d0, 1 %t2 = extractvalue {<8 x i8>, <8 x i8>, <8 x i8>} %d0, 2 %res0 = insertvalue { <8 x i8>, <8 x i8>, <8 x i8> } poison, <8 x i8> %t0, 0 - %res1 = insertvalue { <8 x i8>, <8 x i8>, <8 x i8> } %res0, <8 x i8> %t1, 0 - %res2 = insertvalue { <8 x i8>, <8 x i8>, <8 x i8> } %res1, <8 x i8> %t2, 0 + %res1 = insertvalue { <8 x i8>, <8 x i8>, <8 x i8> } %res0, <8 x i8> %t1, 1 + %res2 = insertvalue { <8 x i8>, <8 x i8>, <8 x i8> } %res1, <8 x i8> %t2, 2 ret { <8 x i8>, <8 x i8>, <8 x i8> } %res2 } diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-segN-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-segN-load.ll index 4eed3df..8c3ebb9 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-segN-load.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-segN-load.ll @@ -1,107 +1,72 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple riscv64 -mattr=+zve64x,+zvl128b < %s | FileCheck %s -define <8 x i8> @load_factor2(ptr %ptr) { +define {<8 x i8>, <8 x i8>} @load_factor2(ptr %ptr) { ; CHECK-LABEL: load_factor2: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vlseg2e8.v v7, (a0) +; CHECK-NEXT: vlseg2e8.v v8, (a0) ; CHECK-NEXT: ret %1 = call { <8 x i8>, <8 x i8> } @llvm.riscv.seg2.load.mask.v8i8.i64(ptr %ptr, <8 x i1> splat (i1 true), i64 8) - %2 = extractvalue { <8 x i8>, <8 x i8> } %1, 0 - %3 = extractvalue { <8 x i8>, <8 x i8> } %1, 1 - ret <8 x i8> %3 + ret {<8 x i8>, <8 x i8>} %1 } -define <8 x i8> @load_factor3(ptr %ptr) { +define {<8 x i8>, <8 x i8>, <8 x i8>} @load_factor3(ptr %ptr) { ; CHECK-LABEL: load_factor3: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vlseg3e8.v v6, (a0) +; CHECK-NEXT: vlseg3e8.v v8, (a0) ; CHECK-NEXT: ret %1 = call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.riscv.seg3.load.mask.v8i8.i64(ptr %ptr, <8 x i1> splat (i1 true), i64 8) - %2 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } %1, 0 - %3 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } %1, 1 - %4 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } %1, 2 - ret <8 x i8> %4 + ret { <8 x i8>, <8 x i8>, <8 x i8> } %1 } -define <8 x i8> @load_factor4(ptr %ptr) { +define {<8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>} @load_factor4(ptr %ptr) { ; CHECK-LABEL: load_factor4: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vlseg4e8.v v5, (a0) +; CHECK-NEXT: vlseg4e8.v v8, (a0) ; CHECK-NEXT: ret %1 = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.riscv.seg4.load.mask.v8i8.i64(ptr %ptr, <8 x i1> splat (i1 true), i64 8) - %2 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 0 - %3 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 1 - %4 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 2 - %5 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 3 - ret <8 x i8> %5 + ret { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1 } -define <8 x i8> @load_factor5(ptr %ptr) { +define {<8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>} @load_factor5(ptr %ptr) { ; CHECK-LABEL: load_factor5: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vlseg5e8.v v4, (a0) +; CHECK-NEXT: vlseg5e8.v v8, (a0) ; CHECK-NEXT: ret %1 = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.riscv.seg5.load.mask.v8i8.i64(ptr %ptr, <8 x i1> splat (i1 true), i64 8) - %2 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 0 - %3 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 1 - %4 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 2 - %5 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 3 - %6 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 4 - ret <8 x i8> %6 + ret { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1 } -define <8 x i8> @load_factor6(ptr %ptr) { +define {<8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>} @load_factor6(ptr %ptr) { ; CHECK-LABEL: load_factor6: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vlseg6e8.v v3, (a0) +; CHECK-NEXT: vlseg6e8.v v8, (a0) ; CHECK-NEXT: ret %1 = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.riscv.seg6.load.mask.v8i8.i64(ptr %ptr, <8 x i1> splat (i1 true), i64 8) - %2 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 0 - %3 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 1 - %4 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 2 - %5 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 3 - %6 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 4 - %7 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 5 - ret <8 x i8> %7 + ret { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1 } -define <8 x i8> @load_factor7(ptr %ptr) { +define {<8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>} @load_factor7(ptr %ptr) { ; CHECK-LABEL: load_factor7: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vlseg7e8.v v2, (a0) +; CHECK-NEXT: vlseg7e8.v v8, (a0) ; CHECK-NEXT: ret %1 = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.riscv.seg7.load.mask.v8i8.i64(ptr %ptr, <8 x i1> splat (i1 true), i64 8) - %2 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 0 - %3 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 1 - %4 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 2 - %5 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 3 - %6 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 4 - %7 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 5 - %8 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 6 - ret <8 x i8> %8 + ret { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1 } -define <8 x i8> @load_factor8(ptr %ptr) { +define {<8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>} @load_factor8(ptr %ptr) { ; CHECK-LABEL: load_factor8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vlseg8e8.v v1, (a0) +; CHECK-NEXT: vlseg8e8.v v8, (a0) ; CHECK-NEXT: ret %1 = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.riscv.seg8.load.mask.v8i8.i64(ptr %ptr, <8 x i1> splat (i1 true), i64 8) - %2 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 0 - %3 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 1 - %4 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 2 - %5 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 3 - %6 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 4 - %7 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 5 - %8 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 6 - %9 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1, 7 - ret <8 x i8> %9 + ret { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %1 } diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll index f9f0aa6..fba592d 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll @@ -372,7 +372,7 @@ define { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @vector_deint ; CHECK-LABEL: vector_deinterleave_load_factor3: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma -; CHECK-NEXT: vlseg3e8.v v6, (a0) +; CHECK-NEXT: vlseg3e8.v v8, (a0) ; CHECK-NEXT: ret %vec = load <vscale x 24 x i8>, ptr %p %d0 = call {<vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>} @llvm.vector.deinterleave3(<vscale x 24 x i8> %vec) @@ -380,8 +380,8 @@ define { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @vector_deint %t1 = extractvalue {<vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>} %d0, 1 %t2 = extractvalue {<vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>} %d0, 2 %res0 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } poison, <vscale x 8 x i8> %t0, 0 - %res1 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res0, <vscale x 8 x i8> %t1, 0 - %res2 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res1, <vscale x 8 x i8> %t2, 0 + %res1 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res0, <vscale x 8 x i8> %t1, 1 + %res2 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res1, <vscale x 8 x i8> %t2, 2 ret { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res2 } @@ -634,3 +634,19 @@ define {<vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i %deinterleaved.results = call {<vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>} @llvm.vector.deinterleave4.nxv32i8(<vscale x 32 x i8> %vec) ret {<vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>} %deinterleaved.results } + +define { <8 x float>, <8 x float> } @deinterleave_unrelated(<16 x float> %arg) { +; CHECK-LABEL: deinterleave_unrelated: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; CHECK-NEXT: vfabs.v v12, v8 +; CHECK-NEXT: li a0, 32 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vnsrl.wx v10, v12, a0 +; CHECK-NEXT: vnsrl.wi v8, v12, 0 +; CHECK-NEXT: ret +entry: + %abs = call <16 x float> @llvm.fabs(<16 x float> %arg) + %res = call { <8 x float>, <8 x float> } @llvm.vector.deinterleave2.v16f32(<16 x float> %abs) + ret { <8 x float>, <8 x float> } %res +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll index 0a96e4f..5b1746d 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll @@ -3744,3 +3744,61 @@ define <vscale x 1 x float> @vector_deinterleave_nxv1f32_nxv8f32_oneactive2(<vsc %ext = extractvalue {<vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>} %res, 5 ret <vscale x 1 x float> %ext } + + +define { <8 x float>, <8 x float> } @interleave_deinterleave2(<8 x float> %a, <8 x float> %b) { +; V-LABEL: interleave_deinterleave2: +; V: # %bb.0: # %entry +; V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; V-NEXT: vwaddu.vv v12, v8, v10 +; V-NEXT: li a0, -1 +; V-NEXT: vwmaccu.vx v12, a0, v10 +; V-NEXT: li a0, 32 +; V-NEXT: vnsrl.wx v10, v12, a0 +; V-NEXT: vnsrl.wi v8, v12, 0 +; V-NEXT: ret +; +; ZIP-LABEL: interleave_deinterleave2: +; ZIP: # %bb.0: # %entry +; ZIP-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; ZIP-NEXT: vmv2r.v v12, v10 +; ZIP-NEXT: li a0, 32 +; ZIP-NEXT: ri.vzip2a.vv v16, v8, v12 +; ZIP-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; ZIP-NEXT: vnsrl.wx v10, v16, a0 +; ZIP-NEXT: vnsrl.wi v8, v16, 0 +; ZIP-NEXT: ret +entry: + %0 = call <16 x float> @llvm.vector.interleave2.v16f32(<8 x float> %a, <8 x float> %b) + %1 = call { <8 x float>, <8 x float> } @llvm.vector.deinterleave2.v16f32(<16 x float> %0) + ret { <8 x float>, <8 x float> } %1 +} + +define <16 x float> @deinterleave_interleave2(<16 x float> %arg) { +; V-LABEL: deinterleave_interleave2: +; V: # %bb.0: # %entry +; V-NEXT: li a0, 32 +; V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; V-NEXT: vnsrl.wi v12, v8, 0 +; V-NEXT: vnsrl.wx v14, v8, a0 +; V-NEXT: vwaddu.vv v8, v12, v14 +; V-NEXT: li a0, -1 +; V-NEXT: vwmaccu.vx v8, a0, v14 +; V-NEXT: ret +; +; ZIP-LABEL: deinterleave_interleave2: +; ZIP: # %bb.0: # %entry +; ZIP-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; ZIP-NEXT: vnsrl.wi v12, v8, 0 +; ZIP-NEXT: li a0, 32 +; ZIP-NEXT: vnsrl.wx v16, v8, a0 +; ZIP-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; ZIP-NEXT: ri.vzip2a.vv v8, v12, v16 +; ZIP-NEXT: ret +entry: + %0 = call { <8 x float>, <8 x float> } @llvm.vector.deinterleave2.v16f32(<16 x float> %arg) + %a = extractvalue { <8 x float>, <8 x float> } %0, 0 + %b = extractvalue { <8 x float>, <8 x float> } %0, 1 + %res = call <16 x float> @llvm.vector.interleave2.v16f32(<8 x float> %a, <8 x float> %b) + ret <16 x float> %res +} diff --git a/llvm/test/CodeGen/WebAssembly/libcall_vectorized.ll b/llvm/test/CodeGen/WebAssembly/libcall_vectorized.ll new file mode 100644 index 0000000..2d1056f --- /dev/null +++ b/llvm/test/CodeGen/WebAssembly/libcall_vectorized.ll @@ -0,0 +1,33 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 + +; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-keep-registers -mattr=+simd128 | FileCheck %s + +target triple = "wasm32-unknown-unknown" + +declare <4 x float> @llvm.exp10.v4f32(<4 x float>) + +define <4 x float> @exp10_f32v4(<4 x float> %v) { +; CHECK-LABEL: exp10_f32v4: +; CHECK: .functype exp10_f32v4 (v128) -> (v128) +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: local.get $push12=, 0 +; CHECK-NEXT: f32x4.extract_lane $push0=, $pop12, 0 +; CHECK-NEXT: call $push1=, exp10f, $pop0 +; CHECK-NEXT: f32x4.splat $push2=, $pop1 +; CHECK-NEXT: local.get $push13=, 0 +; CHECK-NEXT: f32x4.extract_lane $push3=, $pop13, 1 +; CHECK-NEXT: call $push4=, exp10f, $pop3 +; CHECK-NEXT: f32x4.replace_lane $push5=, $pop2, 1, $pop4 +; CHECK-NEXT: local.get $push14=, 0 +; CHECK-NEXT: f32x4.extract_lane $push6=, $pop14, 2 +; CHECK-NEXT: call $push7=, exp10f, $pop6 +; CHECK-NEXT: f32x4.replace_lane $push8=, $pop5, 2, $pop7 +; CHECK-NEXT: local.get $push15=, 0 +; CHECK-NEXT: f32x4.extract_lane $push9=, $pop15, 3 +; CHECK-NEXT: call $push10=, exp10f, $pop9 +; CHECK-NEXT: f32x4.replace_lane $push11=, $pop8, 3, $pop10 +; CHECK-NEXT: return $pop11 +entry: + %r = call <4 x float> @llvm.exp10.v4f32(<4 x float> %v) + ret <4 x float> %r +} diff --git a/llvm/test/CodeGen/WebAssembly/ref-test-func.ll b/llvm/test/CodeGen/WebAssembly/ref-test-func.ll index e4014ba..ea2453f 100644 --- a/llvm/test/CodeGen/WebAssembly/ref-test-func.ll +++ b/llvm/test/CodeGen/WebAssembly/ref-test-func.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; RUN: llc < %s --mtriple=wasm32-unknown-unknown -mcpu=mvp -mattr=+reference-types -verify-machineinstrs | FileCheck --check-prefixes CHECK,CHK32 %s -; RUN: llc < %s --mtriple=wasm64-unknown-unknown -mcpu=mvp -mattr=+reference-types -verify-machineinstrs | FileCheck --check-prefixes CHECK,CHK64 %s +; RUN: llc < %s --mtriple=wasm32-unknown-unknown -mcpu=mvp -mattr=+reference-types -mattr=+gc -verify-machineinstrs | FileCheck --check-prefixes CHECK,CHK32 %s +; RUN: llc < %s --mtriple=wasm64-unknown-unknown -mcpu=mvp -mattr=+reference-types -mattr=+gc -verify-machineinstrs | FileCheck --check-prefixes CHECK,CHK64 %s define void @test_fpsig_void_void(ptr noundef %func) local_unnamed_addr #0 { ; CHECK-LABEL: test_fpsig_void_void: diff --git a/llvm/test/CodeGen/WebAssembly/returned.ll b/llvm/test/CodeGen/WebAssembly/returned.ll index e767e29..aef75d8 100644 --- a/llvm/test/CodeGen/WebAssembly/returned.ll +++ b/llvm/test/CodeGen/WebAssembly/returned.ll @@ -80,3 +80,27 @@ define i32 @test_second_arg(i32 %a, i32 %b) { %call = call i32 @do_something_else(i32 %a, i32 %b) ret i32 %b } + +define void @test() { +; CHECK-LABEL: test: +; CHECK: .functype test () -> () +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get $push0=, __stack_pointer +; CHECK-NEXT: i32.const $push1=, 16 +; CHECK-NEXT: i32.sub $push7=, $pop0, $pop1 +; CHECK-NEXT: local.tee $push6=, $0=, $pop7 +; CHECK-NEXT: global.set __stack_pointer, $pop6 +; CHECK-NEXT: i32.const $push4=, 12 +; CHECK-NEXT: i32.add $push5=, $0, $pop4 +; CHECK-NEXT: call $drop=, returns_arg, $pop5 +; CHECK-NEXT: i32.const $push2=, 16 +; CHECK-NEXT: i32.add $push3=, $0, $pop2 +; CHECK-NEXT: global.set __stack_pointer, $pop3 +; CHECK-NEXT: return +entry: + %a = alloca i32 + call void @llvm.lifetime.start.p0(i64 4, ptr %a) + %ret = call ptr @returns_arg(ptr %a) + call void @llvm.lifetime.end.p0(i64 4, ptr %a) + ret void +} diff --git a/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll b/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll index 1c77ad5..60cfc27 100644 --- a/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll +++ b/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll @@ -70,9 +70,9 @@ target triple = "wasm32-unknown-unknown" ; +call-indirect-overlong, +exception-handling, ; +extended-const, +fp16, +multimemory, +multivalue, ; +mutable-globals, +nontrapping-fptoint, +relaxed-simd, -; +reference-types, +simd128, +sign-ext, +tail-call +; +reference-types, +simd128, +sign-ext, +tail-call, +gc ; BLEEDING-EDGE-LABEL: .section .custom_section.target_features,"",@ -; BLEEDING-EDGE-NEXT: .int8 16 +; BLEEDING-EDGE-NEXT: .int8 17 ; BLEEDING-EDGE-NEXT: .int8 43 ; BLEEDING-EDGE-NEXT: .int8 7 ; BLEEDING-EDGE-NEXT: .ascii "atomics" @@ -95,6 +95,9 @@ target triple = "wasm32-unknown-unknown" ; BLEEDING-EDGE-NEXT: .int8 4 ; BLEEDING-EDGE-NEXT: .ascii "fp16" ; BLEEDING-EDGE-NEXT: .int8 43 +; BLEEDING-EDGE-NEXT: .int8 2 +; BLEEDING-EDGE-NEXT: .ascii "gc" +; BLEEDING-EDGE-NEXT: .int8 43 ; BLEEDING-EDGE-NEXT: .int8 11 ; BLEEDING-EDGE-NEXT: .ascii "multimemory" ; BLEEDING-EDGE-NEXT: .int8 43 diff --git a/llvm/test/CodeGen/X86/embed-bitcode.ll b/llvm/test/CodeGen/X86/embed-bitcode.ll index 0d66ba8..d4af954 100644 --- a/llvm/test/CodeGen/X86/embed-bitcode.ll +++ b/llvm/test/CodeGen/X86/embed-bitcode.ll @@ -1,10 +1,23 @@ ; RUN: llc -filetype=obj -mtriple=x86_64 %s -o %t ; RUN: llvm-readelf -S %t | FileCheck %s +; RUN: llc -filetype=obj -mtriple=x86_64-pc-windows-msvc %s -o %t +; RUN: llvm-readobj -S %t | FileCheck %s --check-prefix=COFF ; CHECK: .text PROGBITS 0000000000000000 [[#%x,OFF:]] 000000 00 AX 0 ; CHECK-NEXT: .llvmbc PROGBITS 0000000000000000 [[#%x,OFF:]] 000004 00 0 ; CHECK-NEXT: .llvmcmd PROGBITS 0000000000000000 [[#%x,OFF:]] 000005 00 0 +; COFF: Name: .llvmbc (2E 6C 6C 76 6D 62 63 00) +; COFF: Characteristics [ +; COFF-NEXT: IMAGE_SCN_ALIGN_1BYTES +; COFF-NEXT: IMAGE_SCN_MEM_DISCARDABLE +; COFF-NEXT: ] +; COFF: Name: .llvmcmd (2E 6C 6C 76 6D 63 6D 64) +; COFF: Characteristics [ +; COFF-NEXT: IMAGE_SCN_ALIGN_1BYTES +; COFF-NEXT: IMAGE_SCN_MEM_DISCARDABLE +; COFF-NEXT: ] + @llvm.embedded.module = private constant [4 x i8] c"BC\C0\DE", section ".llvmbc", align 1 @llvm.cmdline = private constant [5 x i8] c"-cc1\00", section ".llvmcmd", align 1 @llvm.compiler.used = appending global [2 x ptr] [ptr @llvm.embedded.module, ptr @llvm.cmdline], section "llvm.metadata" diff --git a/llvm/test/CodeGen/X86/isel-fpclass.ll b/llvm/test/CodeGen/X86/isel-fpclass.ll new file mode 100644 index 0000000..960bbf5 --- /dev/null +++ b/llvm/test/CodeGen/X86/isel-fpclass.ll @@ -0,0 +1,526 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc < %s -mtriple=i686-linux | FileCheck %s -check-prefixes=X86-SDAGISEL +; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s -check-prefixes=X64,X64-SDAGISEL +; RUN: llc < %s -mtriple=i686-linux -fast-isel -fast-isel-abort=1 | FileCheck %s -check-prefixes=X86-FASTISEL +; RUN: llc < %s -mtriple=x86_64-linux -fast-isel -fast-isel-abort=1 | FileCheck %s -check-prefixes=X64,X64-FASTISEL + +; FIXME: We can reuse/delete llvm/test/CodeGen/X86/is_fpclass.ll when all patches are included. + +define i1 @isnone_f(float %x) { +; X86-SDAGISEL-LABEL: isnone_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: xorl %eax, %eax +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: isnone_f: +; X64: # %bb.0: # %entry +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: isnone_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstp %st(0) +; X86-FASTISEL-NEXT: xorl %eax, %eax +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 0) + ret i1 %0 +} + +define i1 @isany_f(float %x) { +; X86-SDAGISEL-LABEL: isany_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: movb $1, %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: isany_f: +; X64: # %bb.0: # %entry +; X64-NEXT: movb $1, %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: isany_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstp %st(0) +; X86-FASTISEL-NEXT: movb $1, %al +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 1023) + ret i1 %0 +} + +define i1 @issignaling_f(float %x) { +; X86-SDAGISEL-LABEL: issignaling_f: +; X86-SDAGISEL: # %bb.0: +; X86-SDAGISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-SDAGISEL-NEXT: andl {{[0-9]+}}(%esp), %eax +; X86-SDAGISEL-NEXT: cmpl $2143289344, %eax # imm = 0x7FC00000 +; X86-SDAGISEL-NEXT: setl %cl +; X86-SDAGISEL-NEXT: cmpl $2139095041, %eax # imm = 0x7F800001 +; X86-SDAGISEL-NEXT: setge %al +; X86-SDAGISEL-NEXT: andb %cl, %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: issignaling_f: +; X64: # %bb.0: +; X64-NEXT: movd %xmm0, %eax +; X64-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF +; X64-NEXT: cmpl $2143289344, %eax # imm = 0x7FC00000 +; X64-NEXT: setl %cl +; X64-NEXT: cmpl $2139095041, %eax # imm = 0x7F800001 +; X64-NEXT: setge %al +; X64-NEXT: andb %cl, %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: issignaling_f: +; X86-FASTISEL: # %bb.0: +; X86-FASTISEL-NEXT: pushl %eax +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 8 +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstps (%esp) +; X86-FASTISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-FASTISEL-NEXT: andl (%esp), %eax +; X86-FASTISEL-NEXT: cmpl $2143289344, %eax # imm = 0x7FC00000 +; X86-FASTISEL-NEXT: setl %cl +; X86-FASTISEL-NEXT: cmpl $2139095041, %eax # imm = 0x7F800001 +; X86-FASTISEL-NEXT: setge %al +; X86-FASTISEL-NEXT: andb %cl, %al +; X86-FASTISEL-NEXT: popl %ecx +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 4 +; X86-FASTISEL-NEXT: retl + %a0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 1) ; "snan" + ret i1 %a0 +} + + define i1 @isquiet_f(float %x) { +; X86-SDAGISEL-LABEL: isquiet_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-SDAGISEL-NEXT: andl {{[0-9]+}}(%esp), %eax +; X86-SDAGISEL-NEXT: cmpl $2143289344, %eax # imm = 0x7FC00000 +; X86-SDAGISEL-NEXT: setge %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: isquiet_f: +; X64: # %bb.0: # %entry +; X64-NEXT: movd %xmm0, %eax +; X64-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF +; X64-NEXT: cmpl $2143289344, %eax # imm = 0x7FC00000 +; X64-NEXT: setge %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: isquiet_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: pushl %eax +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 8 +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstps (%esp) +; X86-FASTISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-FASTISEL-NEXT: andl (%esp), %eax +; X86-FASTISEL-NEXT: cmpl $2143289344, %eax # imm = 0x7FC00000 +; X86-FASTISEL-NEXT: setge %al +; X86-FASTISEL-NEXT: popl %ecx +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 4 +; X86-FASTISEL-NEXT: retl + entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 2) ; "qnan" + ret i1 %0 +} + +define i1 @not_isquiet_f(float %x) { +; X86-SDAGISEL-LABEL: not_isquiet_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-SDAGISEL-NEXT: andl {{[0-9]+}}(%esp), %eax +; X86-SDAGISEL-NEXT: cmpl $2143289344, %eax # imm = 0x7FC00000 +; X86-SDAGISEL-NEXT: setl %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: not_isquiet_f: +; X64: # %bb.0: # %entry +; X64-NEXT: movd %xmm0, %eax +; X64-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF +; X64-NEXT: cmpl $2143289344, %eax # imm = 0x7FC00000 +; X64-NEXT: setl %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: not_isquiet_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: pushl %eax +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 8 +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstps (%esp) +; X86-FASTISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-FASTISEL-NEXT: andl (%esp), %eax +; X86-FASTISEL-NEXT: cmpl $2143289344, %eax # imm = 0x7FC00000 +; X86-FASTISEL-NEXT: setl %al +; X86-FASTISEL-NEXT: popl %ecx +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 4 +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 1021) ; ~"qnan" + ret i1 %0 +} + +define i1 @isinf_f(float %x) { +; X86-SDAGISEL-LABEL: isinf_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-SDAGISEL-NEXT: andl {{[0-9]+}}(%esp), %eax +; X86-SDAGISEL-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X86-SDAGISEL-NEXT: sete %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: isinf_f: +; X64: # %bb.0: # %entry +; X64-NEXT: movd %xmm0, %eax +; X64-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF +; X64-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X64-NEXT: sete %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: isinf_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: pushl %eax +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 8 +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstps (%esp) +; X86-FASTISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-FASTISEL-NEXT: andl (%esp), %eax +; X86-FASTISEL-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X86-FASTISEL-NEXT: sete %al +; X86-FASTISEL-NEXT: popl %ecx +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 4 +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 516) ; 0x204 = "inf" + ret i1 %0 +} + +define i1 @not_isinf_f(float %x) { +; X86-SDAGISEL-LABEL: not_isinf_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-SDAGISEL-NEXT: andl {{[0-9]+}}(%esp), %eax +; X86-SDAGISEL-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X86-SDAGISEL-NEXT: setne %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: not_isinf_f: +; X64: # %bb.0: # %entry +; X64-NEXT: movd %xmm0, %eax +; X64-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF +; X64-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X64-NEXT: setne %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: not_isinf_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: pushl %eax +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 8 +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstps (%esp) +; X86-FASTISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-FASTISEL-NEXT: andl (%esp), %eax +; X86-FASTISEL-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X86-FASTISEL-NEXT: setne %al +; X86-FASTISEL-NEXT: popl %ecx +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 4 +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 507) ; ~0x204 = "~inf" + ret i1 %0 +} + +define i1 @is_plus_inf_f(float %x) { +; X86-SDAGISEL-LABEL: is_plus_inf_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: cmpl $2139095040, {{[0-9]+}}(%esp) # imm = 0x7F800000 +; X86-SDAGISEL-NEXT: sete %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: is_plus_inf_f: +; X64: # %bb.0: # %entry +; X64-NEXT: movd %xmm0, %eax +; X64-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X64-NEXT: sete %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: is_plus_inf_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: pushl %eax +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 8 +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstps (%esp) +; X86-FASTISEL-NEXT: cmpl $2139095040, (%esp) # imm = 0x7F800000 +; X86-FASTISEL-NEXT: sete %al +; X86-FASTISEL-NEXT: popl %ecx +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 4 +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 512) ; 0x200 = "+inf" + ret i1 %0 +} + +define i1 @is_minus_inf_f(float %x) { +; X86-SDAGISEL-LABEL: is_minus_inf_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: cmpl $-8388608, {{[0-9]+}}(%esp) # imm = 0xFF800000 +; X86-SDAGISEL-NEXT: sete %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: is_minus_inf_f: +; X64: # %bb.0: # %entry +; X64-NEXT: movd %xmm0, %eax +; X64-NEXT: cmpl $-8388608, %eax # imm = 0xFF800000 +; X64-NEXT: sete %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: is_minus_inf_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: pushl %eax +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 8 +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstps (%esp) +; X86-FASTISEL-NEXT: cmpl $-8388608, (%esp) # imm = 0xFF800000 +; X86-FASTISEL-NEXT: sete %al +; X86-FASTISEL-NEXT: popl %ecx +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 4 +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 4) ; "-inf" + ret i1 %0 +} + +define i1 @not_is_minus_inf_f(float %x) { +; X86-SDAGISEL-LABEL: not_is_minus_inf_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: cmpl $-8388608, {{[0-9]+}}(%esp) # imm = 0xFF800000 +; X86-SDAGISEL-NEXT: setne %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: not_is_minus_inf_f: +; X64: # %bb.0: # %entry +; X64-NEXT: movd %xmm0, %eax +; X64-NEXT: cmpl $-8388608, %eax # imm = 0xFF800000 +; X64-NEXT: setne %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: not_is_minus_inf_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: pushl %eax +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 8 +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstps (%esp) +; X86-FASTISEL-NEXT: cmpl $-8388608, (%esp) # imm = 0xFF800000 +; X86-FASTISEL-NEXT: setne %al +; X86-FASTISEL-NEXT: popl %ecx +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 4 +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 1019) ; ~"-inf" + ret i1 %0 +} + +define i1 @isfinite_f(float %x) { +; X86-SDAGISEL-LABEL: isfinite_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-SDAGISEL-NEXT: andl {{[0-9]+}}(%esp), %eax +; X86-SDAGISEL-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X86-SDAGISEL-NEXT: setl %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: isfinite_f: +; X64: # %bb.0: # %entry +; X64-NEXT: movd %xmm0, %eax +; X64-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF +; X64-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X64-NEXT: setl %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: isfinite_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: pushl %eax +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 8 +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstps (%esp) +; X86-FASTISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-FASTISEL-NEXT: andl (%esp), %eax +; X86-FASTISEL-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X86-FASTISEL-NEXT: setl %al +; X86-FASTISEL-NEXT: popl %ecx +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 4 +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 504) ; 0x1f8 = "finite" + ret i1 %0 +} + +define i1 @not_isfinite_f(float %x) { +; X86-SDAGISEL-LABEL: not_isfinite_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-SDAGISEL-NEXT: andl {{[0-9]+}}(%esp), %eax +; X86-SDAGISEL-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X86-SDAGISEL-NEXT: setge %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: not_isfinite_f: +; X64: # %bb.0: # %entry +; X64-NEXT: movd %xmm0, %eax +; X64-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF +; X64-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X64-NEXT: setge %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: not_isfinite_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: pushl %eax +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 8 +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstps (%esp) +; X86-FASTISEL-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF +; X86-FASTISEL-NEXT: andl (%esp), %eax +; X86-FASTISEL-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X86-FASTISEL-NEXT: setge %al +; X86-FASTISEL-NEXT: popl %ecx +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 4 +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 519) ; ~0x1f8 = "~finite" + ret i1 %0 +} + +define i1 @is_plus_finite_f(float %x) { +; X86-SDAGISEL-LABEL: is_plus_finite_f: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: cmpl $2139095040, {{[0-9]+}}(%esp) # imm = 0x7F800000 +; X86-SDAGISEL-NEXT: setb %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: is_plus_finite_f: +; X64: # %bb.0: # %entry +; X64-NEXT: movd %xmm0, %eax +; X64-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 +; X64-NEXT: setb %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: is_plus_finite_f: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: pushl %eax +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 8 +; X86-FASTISEL-NEXT: flds {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstps (%esp) +; X86-FASTISEL-NEXT: cmpl $2139095040, (%esp) # imm = 0x7F800000 +; X86-FASTISEL-NEXT: setb %al +; X86-FASTISEL-NEXT: popl %ecx +; X86-FASTISEL-NEXT: .cfi_def_cfa_offset 4 +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 448) ; 0x1c0 = "+finite" + ret i1 %0 +} + +define i1 @isnone_d(double %x) nounwind { +; X86-SDAGISEL-LABEL: isnone_d: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: xorl %eax, %eax +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: isnone_d: +; X64: # %bb.0: # %entry +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: isnone_d: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: fldl {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstp %st(0) +; X86-FASTISEL-NEXT: xorl %eax, %eax +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f64(double %x, i32 0) + ret i1 %0 +} + +define i1 @isany_d(double %x) nounwind { +; X86-SDAGISEL-LABEL: isany_d: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: movb $1, %al +; X86-SDAGISEL-NEXT: retl +; +; X64-LABEL: isany_d: +; X64: # %bb.0: # %entry +; X64-NEXT: movb $1, %al +; X64-NEXT: retq +; +; X86-FASTISEL-LABEL: isany_d: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: fldl {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstp %st(0) +; X86-FASTISEL-NEXT: movb $1, %al +; X86-FASTISEL-NEXT: retl +entry: + %0 = tail call i1 @llvm.is.fpclass.f64(double %x, i32 1023) + ret i1 %0 +} + +define i1 @isnone_f80(x86_fp80 %x) nounwind { +; X86-SDAGISEL-LABEL: isnone_f80: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: xorl %eax, %eax +; X86-SDAGISEL-NEXT: retl +; +; X64-SDAGISEL-LABEL: isnone_f80: +; X64-SDAGISEL: # %bb.0: # %entry +; X64-SDAGISEL-NEXT: xorl %eax, %eax +; X64-SDAGISEL-NEXT: retq +; +; X86-FASTISEL-LABEL: isnone_f80: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: fldt {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstp %st(0) +; X86-FASTISEL-NEXT: xorl %eax, %eax +; X86-FASTISEL-NEXT: retl +; +; X64-FASTISEL-LABEL: isnone_f80: +; X64-FASTISEL: # %bb.0: # %entry +; X64-FASTISEL-NEXT: fldt {{[0-9]+}}(%rsp) +; X64-FASTISEL-NEXT: fstp %st(0) +; X64-FASTISEL-NEXT: xorl %eax, %eax +; X64-FASTISEL-NEXT: retq +entry: +%0 = tail call i1 @llvm.is.fpclass.f80(x86_fp80 %x, i32 0) +ret i1 %0 +} + +define i1 @isany_f80(x86_fp80 %x) nounwind { +; X86-SDAGISEL-LABEL: isany_f80: +; X86-SDAGISEL: # %bb.0: # %entry +; X86-SDAGISEL-NEXT: movb $1, %al +; X86-SDAGISEL-NEXT: retl +; +; X64-SDAGISEL-LABEL: isany_f80: +; X64-SDAGISEL: # %bb.0: # %entry +; X64-SDAGISEL-NEXT: movb $1, %al +; X64-SDAGISEL-NEXT: retq +; +; X86-FASTISEL-LABEL: isany_f80: +; X86-FASTISEL: # %bb.0: # %entry +; X86-FASTISEL-NEXT: fldt {{[0-9]+}}(%esp) +; X86-FASTISEL-NEXT: fstp %st(0) +; X86-FASTISEL-NEXT: movb $1, %al +; X86-FASTISEL-NEXT: retl +; +; X64-FASTISEL-LABEL: isany_f80: +; X64-FASTISEL: # %bb.0: # %entry +; X64-FASTISEL-NEXT: fldt {{[0-9]+}}(%rsp) +; X64-FASTISEL-NEXT: fstp %st(0) +; X64-FASTISEL-NEXT: movb $1, %al +; X64-FASTISEL-NEXT: retq +entry: + %0 = tail call i1 @llvm.is.fpclass.f80(x86_fp80 %x, i32 1023) + ret i1 %0 +} diff --git a/llvm/test/CodeGen/X86/late-tail-dup-computed-goto.mir b/llvm/test/CodeGen/X86/late-tail-dup-computed-goto.mir new file mode 100644 index 0000000..e272e7e --- /dev/null +++ b/llvm/test/CodeGen/X86/late-tail-dup-computed-goto.mir @@ -0,0 +1,128 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5 +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=tailduplication -tail-dup-pred-size=1 -tail-dup-succ-size=1 %s -o - | FileCheck %s +# +# Check that only the computed gotos are duplicated aggressively. +# +--- | + @computed_goto.dispatch = constant [5 x ptr] [ptr null, ptr blockaddress(@computed_goto, %bb1), ptr blockaddress(@computed_goto, %bb2), ptr blockaddress(@computed_goto, %bb3), ptr blockaddress(@computed_goto, %bb4)] + declare i64 @f0() + declare i64 @f1() + declare i64 @f2() + declare i64 @f3() + declare i64 @f4() + declare i64 @f5() + define void @computed_goto() { + start: + ret void + bb1: + ret void + bb2: + ret void + bb3: + ret void + bb4: + ret void + } + define void @jump_table() { ret void } + define void @jump_table_pic() { ret void } +... +--- +name: computed_goto +alignment: 1 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +failedISel: false +tracksRegLiveness: true +body: | + ; CHECK-LABEL: name: computed_goto + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: CALL64pcrel32 target-flags(x86-plt) @f0, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64_nosp = COPY [[COPY]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64_nosp = COPY [[COPY1]] + ; CHECK-NEXT: JMP64m $noreg, 8, [[COPY2]], @computed_goto.dispatch, $noreg + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.bb1 (ir-block-address-taken %ir-block.bb1): + ; CHECK-NEXT: successors: %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: CALL64pcrel32 target-flags(x86-plt) @f1, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $rax + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64_nosp = COPY [[COPY3]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64_nosp = COPY [[COPY1]] + ; CHECK-NEXT: JMP64m $noreg, 8, [[COPY2]], @computed_goto.dispatch, $noreg + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.bb2 (ir-block-address-taken %ir-block.bb2): + ; CHECK-NEXT: successors: %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: CALL64pcrel32 target-flags(x86-plt) @f2, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $rax + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64_nosp = COPY [[COPY4]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64_nosp = COPY [[COPY1]] + ; CHECK-NEXT: JMP64m $noreg, 8, [[COPY2]], @computed_goto.dispatch, $noreg + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.3.bb3 (ir-block-address-taken %ir-block.bb3): + ; CHECK-NEXT: successors: %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: CALL64pcrel32 target-flags(x86-plt) @f3, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY $rax + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64_nosp = COPY [[COPY5]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64_nosp = COPY [[COPY1]] + ; CHECK-NEXT: JMP64m $noreg, 8, [[COPY2]], @computed_goto.dispatch, $noreg + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.4.bb4 (ir-block-address-taken %ir-block.bb4): + ; CHECK-NEXT: successors: %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: CALL64pcrel32 target-flags(x86-plt) @f4, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gr64 = COPY $rax + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64_nosp = COPY [[COPY6]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64_nosp = COPY [[COPY1]] + ; CHECK-NEXT: JMP64m $noreg, 8, [[COPY2]], @computed_goto.dispatch, $noreg + bb.0: + successors: %bb.5(0x80000000) + + CALL64pcrel32 target-flags(x86-plt) @f0, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax + %0:gr64 = COPY $rax + %6:gr64_nosp = COPY %0 + JMP_1 %bb.5 + + bb.1.bb1 (ir-block-address-taken %ir-block.bb1): + successors: %bb.5(0x80000000) + + CALL64pcrel32 target-flags(x86-plt) @f1, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax + %1:gr64 = COPY $rax + %6:gr64_nosp = COPY %1 + JMP_1 %bb.5 + + bb.2.bb2 (ir-block-address-taken %ir-block.bb2): + successors: %bb.5(0x80000000) + + CALL64pcrel32 target-flags(x86-plt) @f2, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax + %2:gr64 = COPY $rax + %6:gr64_nosp = COPY %2 + JMP_1 %bb.5 + + bb.3.bb3 (ir-block-address-taken %ir-block.bb3): + successors: %bb.5(0x80000000) + + CALL64pcrel32 target-flags(x86-plt) @f3, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax + %3:gr64 = COPY $rax + %6:gr64_nosp = COPY %3 + JMP_1 %bb.5 + + bb.4.bb4 (ir-block-address-taken %ir-block.bb4): + successors: %bb.5(0x80000000) + + CALL64pcrel32 target-flags(x86-plt) @f4, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax + %4:gr64 = COPY $rax + %6:gr64_nosp = COPY %4 + + bb.5: + successors: %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000) + + %5:gr64_nosp = COPY %6 + JMP64m $noreg, 8, %5, @computed_goto.dispatch, $noreg +... diff --git a/llvm/test/CodeGen/XCore/section-name.ll b/llvm/test/CodeGen/XCore/section-name.ll index 0fa2cc6..b2176ec 100644 --- a/llvm/test/CodeGen/XCore/section-name.ll +++ b/llvm/test/CodeGen/XCore/section-name.ll @@ -1,4 +1,4 @@ -; RUN: not llc < %s -mtriple=xcore -o /dev/null 2>&1 | FileCheck %s +; RUN: llc < %s -mtriple=xcore | FileCheck %s @bar = internal global i32 zeroinitializer @@ -6,4 +6,4 @@ define void @".dp.bss"() { ret void } -; CHECK: <unknown>:0: error: symbol '.dp.bss' is already defined +; CHECK: .dp.bss: |